From 9a2d329e1b2ab456a061a15f49acf62155d46e46 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 29 Jul 2026 19:36:28 +0200 Subject: [PATCH 01/54] fix order --- Common/include/option_structure.hpp | 1 + Common/src/CConfig.cpp | 2 + Common/src/geometry/CMultiGridGeometry.cpp | 230 ++++++++++++--------- config_template.cfg | 5 + 4 files changed, 141 insertions(+), 97 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 1ba1bab61fe..5eda82d9fde 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1126,6 +1126,7 @@ struct CMGOptions { su2double MG_Smooth_StagnationTol{0.0}; /*!< \brief Stagnation early exit: stop if current_rms >= prev_rms * tol. 0 = disabled. */ bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ + bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 2c0dce07e68..09d82d8574a 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2069,6 +2069,8 @@ void CConfig::SetConfig_Options() { addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); /*!\brief MG_IMPLICIT_LINES_MAX_LENGTH\n DESCRIPTION: Maximum number of nodes on a wall-normal implicit agglomeration line (including the wall seed node). DEFAULT: 20 \ingroup Config*/ addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 20); + /*!\brief MG_IMPLICIT_LINES_ISOTROPIC\n DESCRIPTION: Use isotropic agglomeration along implicit lines (4 cells per coarse CV) instead of anisotropic (2 cells per coarse CV). DEFAULT: NO \ingroup Config*/ + addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 684da742b13..79cd4a6b609 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1286,6 +1286,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop line if direction deviates more than this. */ const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); + const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; const unsigned long nPointFine = fine_grid->GetnPoint(); @@ -1381,14 +1382,20 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (rank == MASTER_NODE) { cout << "Implicit line agglomeration: detected " << lines.size() << " lines." << endl; + cout << " Mode: " << (ISOTROPIC ? "ISOTROPIC" : "ANISOTROPIC") << endl; } - /*--- Advancing-front greedy pairing with cross-line merging. - * For each pair stage k, process interior positions (1+2k, 1+2k+1). - * When two lines share the same wall-node parent CV, merge their pairs - * into a single 4-child coarse CV. Otherwise create 2-child coarse CVs. ---*/ + /*--- Agglomeration strategy: + * ANISOTROPIC (default): Pair nodes at the SAME distance from wall on DIFFERENT lines. + * Each coarse CV has 2 fine children (from adjacent lines). + * Reduces mesh by factor ~2 normal to wall, preserves resolution along wall. + * + * ISOTROPIC: Group 4 nodes (2 positions × 2 lines) into one coarse CV. + * Each coarse CV has 4 fine children. + * Reduces mesh uniformly by factor ~4 in all directions. + ---*/ vector reserved(nPointFine, 0); - unsigned pair_idx = 0; + unsigned position_idx = 0; while (true) { bool any_work = false; @@ -1399,111 +1406,140 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, for (unsigned long li = 0; li < lines.size(); ++li) { const auto& L = lines[li]; if (L.empty()) continue; - const auto idx2 = 1 + 2 * pair_idx + 1; - if (L.size() <= idx2) continue; // no pair at this stage + if (ISOTROPIC) { + const auto idx2 = 1 + 2 * position_idx + 1; + if (L.size() <= idx2) continue; // no pair at this stage + } else { + if (L.size() <= 1 + position_idx) continue; // no position at this index + } const auto pW = fine_grid->nodes->GetParent_CV(L[0]); parent_to_lines[pW].push_back(li); } vector line_processed(lines.size(), 0); - /*--- A) Cross-line merges: parents with multiple lines ---*/ - for (auto& [parent, line_ids] : parent_to_lines) { - if (line_ids.size() < 2) continue; - - for (size_t k = 0; k + 1 < line_ids.size(); k += 2) { - const auto li1 = line_ids[k]; - const auto li2 = line_ids[k + 1]; - if (line_processed[li1] || line_processed[li2]) continue; - - const auto& L1 = lines[li1]; - const auto& L2 = lines[li2]; - const auto idx1 = 1 + 2 * pair_idx; - const auto idx2 = idx1 + 1; - if (L1.size() <= idx2 || L2.size() <= idx2) continue; - - const auto a = L1[idx1], b = L1[idx2]; - const auto c = L2[idx1], d = L2[idx2]; - - /*--- Skip if any node is already claimed ---*/ - if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b) || - fine_grid->nodes->GetAgglomerate(c) || fine_grid->nodes->GetAgglomerate(d)) - continue; - if (reserved[a] || reserved[b] || reserved[c] || reserved[d]) continue; - - /*--- Geometrical quality check ---*/ - if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config) || - !GeometricalCheck(c, fine_grid, config) || !GeometricalCheck(d, fine_grid, config)) - continue; - - /*--- Guard against duplicate indices ---*/ - if (a == b || a == c || a == d || b == c || b == d || c == d) { - for (auto other_li : line_ids) line_processed[other_li] = 1; - continue; - } - - /*--- Create 4-child coarse CV ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - fine_grid->nodes->SetParent_CV(c, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 2, c); - fine_grid->nodes->SetParent_CV(d, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 3, d); - nodes->SetnChildren_CV(Index_CoarseCV, 4); - - reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - MGQueue_InnerCV.RemoveCV(c); - MGQueue_InnerCV.RemoveCV(d); + if (ISOTROPIC) { + /*--- ISOTROPIC MODE: Group 4 children per coarse CV (2 positions × 2 lines) ---*/ + for (auto& [parent, line_ids] : parent_to_lines) { + if (line_ids.size() < 2) continue; + + for (size_t k = 0; k + 1 < line_ids.size(); k += 2) { + const auto li1 = line_ids[k]; + const auto li2 = line_ids[k + 1]; + if (line_processed[li1] || line_processed[li2]) continue; + + const auto& L1 = lines[li1]; + const auto& L2 = lines[li2]; + const auto idx1 = 1 + 2 * position_idx; + const auto idx2 = idx1 + 1; + if (L1.size() <= idx2 || L2.size() <= idx2) continue; + + const auto a = L1[idx1], b = L1[idx2]; + const auto c = L2[idx1], d = L2[idx2]; + + /*--- Skip if any node is already claimed ---*/ + if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b) || + fine_grid->nodes->GetAgglomerate(c) || fine_grid->nodes->GetAgglomerate(d)) + continue; + if (reserved[a] || reserved[b] || reserved[c] || reserved[d]) continue; + + /*--- Geometrical quality check ---*/ + if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config) || + !GeometricalCheck(c, fine_grid, config) || !GeometricalCheck(d, fine_grid, config)) + continue; + + /*--- Guard against duplicate indices ---*/ + if (a == b || a == c || a == d || b == c || b == d || c == d) { + for (auto other_li : line_ids) line_processed[other_li] = 1; + continue; + } - Index_CoarseCV++; - line_processed[li1] = line_processed[li2] = 1; - for (auto other_li : line_ids) - if (other_li != li1 && other_li != li2) line_processed[other_li] = 1; - any_work = true; + /*--- Create 4-child coarse CV (isotropic agglomeration) ---*/ + fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 0, a); + fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 1, b); + fine_grid->nodes->SetParent_CV(c, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 2, c); + fine_grid->nodes->SetParent_CV(d, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 3, d); + nodes->SetnChildren_CV(Index_CoarseCV, 4); + + reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; + MGQueue_InnerCV.RemoveCV(a); + MGQueue_InnerCV.RemoveCV(b); + MGQueue_InnerCV.RemoveCV(c); + MGQueue_InnerCV.RemoveCV(d); + + Index_CoarseCV++; + line_processed[li1] = line_processed[li2] = 1; + for (auto other_li : line_ids) + if (other_li != li1 && other_li != li2) line_processed[other_li] = 1; + any_work = true; + } + } + } else { + /*--- ANISOTROPIC MODE: Pair nodes at SAME position on DIFFERENT lines ---*/ + for (auto& [parent, line_ids] : parent_to_lines) { + if (line_ids.size() < 2) continue; + + /*--- Pair consecutive lines at the same position ---*/ + for (size_t k = 0; k + 1 < line_ids.size(); k += 2) { + const auto li1 = line_ids[k]; + const auto li2 = line_ids[k + 1]; + if (line_processed[li1] || line_processed[li2]) continue; + + const auto& L1 = lines[li1]; + const auto& L2 = lines[li2]; + const auto pos = 1 + position_idx; + if (L1.size() <= pos || L2.size() <= pos) continue; + + const auto a = L1[pos]; + const auto b = L2[pos]; + + /*--- Skip if any node is already claimed ---*/ + if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b)) continue; + if (reserved[a] || reserved[b]) continue; + + /*--- Geometrical quality check ---*/ + if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config)) continue; + + /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ + fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 0, a); + fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 1, b); + nodes->SetnChildren_CV(Index_CoarseCV, 2); + + reserved[a] = reserved[b] = 1; + MGQueue_InnerCV.RemoveCV(a); + MGQueue_InnerCV.RemoveCV(b); + + Index_CoarseCV++; + line_processed[li1] = line_processed[li2] = 1; + any_work = true; + } } } - /*--- B) Single-line 2-child merges for remaining lines ---*/ - for (unsigned long li = 0; li < lines.size(); ++li) { - if (line_processed[li]) continue; - const auto& L = lines[li]; - const auto idx1 = 1 + 2 * pair_idx; - const auto idx2 = idx1 + 1; - if (L.size() <= idx2) continue; - - const auto a = L[idx1], b = L[idx2]; - if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b)) continue; - if (reserved[a] || reserved[b]) continue; - if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config)) continue; - - /*--- Create 2-child coarse CV ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - nodes->SetnChildren_CV(Index_CoarseCV, 2); - - reserved[a] = reserved[b] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - - Index_CoarseCV++; - any_work = true; - } - - pair_idx++; + position_idx++; if (!any_work) break; - /*--- Check if any line still has pairs at the next stage ---*/ + /*--- Check if any line still has positions available ---*/ bool any_more = false; - for (const auto& L : lines) { - if (L.size() > 1 + 2 * pair_idx + 1) { - any_more = true; - break; + if (ISOTROPIC) { + for (const auto& L : lines) { + if (L.size() > 1 + 2 * position_idx + 1) { + any_more = true; + break; + } + } + } else { + for (const auto& L : lines) { + if (L.size() > 1 + position_idx) { + any_more = true; + break; + } } } if (!any_more) break; diff --git a/config_template.cfg b/config_template.cfg index a7d357240e0..9e2fe09703d 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1719,6 +1719,11 @@ MG_IMPLICIT_LINES= NO % Maximum nodes on a wall-normal implicit agglomeration line, including the wall seed. % Increase to extend the line deeper into the boundary layer (default 20). MG_IMPLICIT_LINES_MAX_LENGTH= 20 +% +% Use isotropic (vs anisotropic) agglomeration for implicit lines (NO, YES) +% Anisotropic (NO): Pair cells normal to wall (2 cells per coarse CV, reduces mesh ~2x) +% Isotropic (YES): Pair cells in all directions (4 cells per coarse CV, reduces mesh ~4x) +MG_IMPLICIT_LINES_ISOTROPIC= NO % -------------------------- MESH SMOOTHING -----------------------------% % From 06790fb70c60c845d9f0c3deeb2faccd78eb4300 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Thu, 30 Jul 2026 20:18:30 +0200 Subject: [PATCH 02/54] minor implicit line changes --- Common/include/option_structure.hpp | 1 + Common/src/CConfig.cpp | 2 + Common/src/geometry/CMultiGridGeometry.cpp | 464 ++++++++++++++---- .../src/integration/CMultiGridIntegration.cpp | 11 +- config_template.cfg | 4 + 5 files changed, 371 insertions(+), 111 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 5eda82d9fde..45b2a949b0b 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1127,6 +1127,7 @@ struct CMGOptions { bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ + unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 09d82d8574a..a0bcc8f5621 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2071,6 +2071,8 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 20); /*!\brief MG_IMPLICIT_LINES_ISOTROPIC\n DESCRIPTION: Use isotropic agglomeration along implicit lines (4 cells per coarse CV) instead of anisotropic (2 cells per coarse CV). DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); + /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ + addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 79cd4a6b609..c9192e17afd 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -312,8 +312,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } /*--- Agglomerate high-aspect-ratio interior nodes along implicit lines from walls. ---*/ + unsigned long Index_CoarseCV_before_implicit_lines = Index_CoarseCV; + unsigned long Index_CoarseCV_after_implicit_lines = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, MGQueue_InnerCV); + Index_CoarseCV_after_implicit_lines = Index_CoarseCV; } /*--- STEP 2: Agglomerate the domain points. ---*/ @@ -428,6 +431,42 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nPointDomain = Index_CoarseCV; nPoint = nPointDomain; + /*--- DIAGNOSTIC: Check CV child counts after domain agglomeration ---*/ + if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { + unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; + unsigned long n_corrupted_implicit_CVs = 0; + for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { + const auto nChildren = nodes->GetnChildren_CV(iCV); + if (nChildren == 1) nCVs_1child++; + else if (nChildren == 2) nCVs_2child++; + else if (nChildren == 3) nCVs_3child++; + else if (nChildren == 4) nCVs_4child++; + else nCVs_other++; + + if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { + n_corrupted_implicit_CVs++; + if (n_corrupted_implicit_CVs <= 5) { + cout << " CORRUPTION DETECTED in CV " << iCV << ": has " << nChildren << " children (expected 2)" << endl; + cout << " Children nodes: "; + for (unsigned short iChild = 0; iChild < nChildren; iChild++) { + cout << nodes->GetChildren_CV(iCV, iChild); + if (iChild < nChildren - 1) cout << ", "; + } + cout << endl; + } + } + } + if (n_corrupted_implicit_CVs > 0) { + cout << " AFTER DOMAIN AGGLOMERATION: " << n_corrupted_implicit_CVs + << " implicit line CVs were corrupted (child count != 2)" << endl; + cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child + << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child + << ", 4-child=" << nCVs_4child; + if (nCVs_other > 0) cout << ", other=" << nCVs_other; + cout << endl; + } + } + /*--- Check that there are no hanging nodes. Detect isolated points (only 1 neighbor), and merge their children CV's with the neighbor. ---*/ @@ -501,6 +540,58 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- Diagnostic: Check if implicit line CVs were corrupted by hanging node correction ---*/ + if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { + unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; + unsigned long n_corrupted_after_hanging = 0; + for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { + const auto nChildren = nodes->GetnChildren_CV(iCV); + if (nChildren == 1) nCVs_1child++; + else if (nChildren == 2) nCVs_2child++; + else if (nChildren == 3) nCVs_3child++; + else if (nChildren == 4) nCVs_4child++; + else nCVs_other++; + + if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { + n_corrupted_after_hanging++; + } + } + if (n_corrupted_after_hanging > 0) { + cout << " AFTER HANGING NODE CORRECTION: " << n_corrupted_after_hanging + << " implicit line CVs corrupted (child count != 2)" << endl; + cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child + << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child + << ", 4-child=" << nCVs_4child; + if (nCVs_other > 0) cout << ", other=" << nCVs_other; + cout << endl; + } + } + + /*--- Final summary of all CVs ---*/ + if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { + cout << " Expected ratio: ~2 nodes per CV (actual: " << fixed << setprecision(2) + << (double)fine_grid->GetnPoint() / (double)nPointDomain << ")" << endl; + + unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; + for (auto iCV = 0ul; iCV < nPointDomain; iCV++) { + const auto nChildren = nodes->GetnChildren_CV(iCV); + if (nChildren == 1) nCVs_1child++; + else if (nChildren == 2) nCVs_2child++; + else if (nChildren == 3) nCVs_3child++; + else if (nChildren == 4) nCVs_4child++; + else nCVs_other++; + } + cout << " CV distribution: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child + << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; + if (nCVs_other > 0) cout << ", other=" << nCVs_other; + cout << endl; + + if (nCVs_3child > 0 || (!config->GetMGOptions().MG_Implicit_Lines_Isotropic && nCVs_4child > 0)) { + cout << " WARNING: Detected unexpected CV child counts (3-child=" << nCVs_3child + << ", 4-child=" << nCVs_4child << " in ANISO mode)" << endl; + } + } + /*--- Reset the neighbor information. ---*/ nodes->ResetPoints(); @@ -659,11 +750,9 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un SetGlobal_nPointDomain(Global_nPointCoarse); if (iMesh != MESH_0) { - /*--- Note: CFL at the coarse levels have a large impact on convergence, - this should be rewritten to use adaptive CFL. ---*/ - const su2double Coeff = 1.5; - const su2double CFL = config->GetCFL(iMesh - 1) / Coeff; - config->SetCFL(iMesh, CFL); + /*--- Initialize coarse-level CFL from config. MG_CFL_SCALING will + apply per-level reductions during the multigrid cycle. ---*/ + config->SetCFL(iMesh, config->GetCFL(MESH_0)); } const su2double ratio = su2double(Global_nPointFine) / su2double(Global_nPointCoarse); @@ -1289,6 +1378,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; const unsigned long nPointFine = fine_grid->GetnPoint(); + const unsigned long starting_Index_CoarseCV = Index_CoarseCV; /*--- Track how many CVs we create ---*/ + const bool DEBUG_OUTPUT = (rank == MASTER_NODE); /*--- Enable detailed diagnostic output ---*/ + const unsigned long DEBUG_CV_LIMIT = 20; /*--- Show details for first N CVs ---*/ /*--- Collect implicit lines starting at viscous (no-slip) wall vertices only. * Seeding from non-wall boundaries (farfield, inlet, outlet, symmetry) would @@ -1383,6 +1475,28 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (rank == MASTER_NODE) { cout << "Implicit line agglomeration: detected " << lines.size() << " lines." << endl; cout << " Mode: " << (ISOTROPIC ? "ISOTROPIC" : "ANISOTROPIC") << endl; + /*--- Show line length distribution ---*/ + size_t min_len = ULONG_MAX, max_len = 0; + su2double avg_len = 0.0; + for (const auto& L : lines) { + min_len = min(min_len, L.size()); + max_len = max(max_len, L.size()); + avg_len += L.size(); + } + if (!lines.empty()) avg_len /= lines.size(); + cout << " Line lengths: min=" << min_len << ", max=" << max_len << ", avg=" << std::setprecision(1) << std::fixed << avg_len << endl; + + /*--- Show first few lines for debugging ---*/ + cout << " First 5 lines (showing first 4 nodes):" << endl; + for (size_t i = 0; i < min(size_t(5), lines.size()); ++i) { + cout << " Line " << i << " (len=" << lines[i].size() << "): ["; + for (size_t j = 0; j < min(size_t(4), lines[i].size()); ++j) { + if (j > 0) cout << ", "; + cout << lines[i][j]; + } + if (lines[i].size() > 4) cout << ", ..."; + cout << "]" << endl; + } } /*--- Agglomeration strategy: @@ -1399,10 +1513,11 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, while (true) { bool any_work = false; + vector line_processed(lines.size(), 0); - /*--- Build map: wall parent CV -> list of line indices ---*/ - unordered_map> parent_to_lines; - parent_to_lines.reserve(lines.size()); + /*--- Build list of active lines (have nodes at current position) ---*/ + vector active_lines; + active_lines.reserve(lines.size()); for (unsigned long li = 0; li < lines.size(); ++li) { const auto& L = lines[li]; if (L.empty()) continue; @@ -1412,113 +1527,192 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } else { if (L.size() <= 1 + position_idx) continue; // no position at this index } - const auto pW = fine_grid->nodes->GetParent_CV(L[0]); - parent_to_lines[pW].push_back(li); + active_lines.push_back(li); } - vector line_processed(lines.size(), 0); - if (ISOTROPIC) { - /*--- ISOTROPIC MODE: Group 4 children per coarse CV (2 positions × 2 lines) ---*/ - for (auto& [parent, line_ids] : parent_to_lines) { - if (line_ids.size() < 2) continue; - - for (size_t k = 0; k + 1 < line_ids.size(); k += 2) { - const auto li1 = line_ids[k]; - const auto li2 = line_ids[k + 1]; - if (line_processed[li1] || line_processed[li2]) continue; - - const auto& L1 = lines[li1]; - const auto& L2 = lines[li2]; - const auto idx1 = 1 + 2 * position_idx; - const auto idx2 = idx1 + 1; - if (L1.size() <= idx2 || L2.size() <= idx2) continue; - - const auto a = L1[idx1], b = L1[idx2]; - const auto c = L2[idx1], d = L2[idx2]; - - /*--- Skip if any node is already claimed ---*/ - if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b) || - fine_grid->nodes->GetAgglomerate(c) || fine_grid->nodes->GetAgglomerate(d)) - continue; - if (reserved[a] || reserved[b] || reserved[c] || reserved[d]) continue; - - /*--- Geometrical quality check ---*/ - if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config) || - !GeometricalCheck(c, fine_grid, config) || !GeometricalCheck(d, fine_grid, config)) - continue; - - /*--- Guard against duplicate indices ---*/ - if (a == b || a == c || a == d || b == c || b == d || c == d) { - for (auto other_li : line_ids) line_processed[other_li] = 1; - continue; + /*--- ISOTROPIC MODE: Group 4 children per coarse CV (2 positions × 2 lines) + Use spatial neighbor search to pair adjacent lines. ---*/ + for (auto li1 : active_lines) { + if (line_processed[li1]) continue; + + const auto& L1 = lines[li1]; + const auto idx1 = 1 + 2 * position_idx; + const auto idx2 = idx1 + 1; + if (L1.size() <= idx2) continue; + + const auto a = L1[idx1], b = L1[idx2]; + if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b)) continue; + if (reserved[a] || reserved[b]) continue; + + /*--- Find nearest neighbor line by checking mesh neighbors of node 'a' ---*/ + unsigned long li2_best = std::numeric_limits::max(); + for (auto neighbor_point : fine_grid->nodes->GetPoints(a)) { + /*--- Check if this neighbor belongs to another unprocessed line at same position ---*/ + for (auto li2 : active_lines) { + if (li2 == li1 || line_processed[li2]) continue; + const auto& L2 = lines[li2]; + if (L2.size() <= idx2) continue; + const auto c = L2[idx1]; + if (c == neighbor_point) { + li2_best = li2; + break; + } } + if (li2_best != std::numeric_limits::max()) break; + } + + if (li2_best == std::numeric_limits::max()) continue; - /*--- Create 4-child coarse CV (isotropic agglomeration) ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - fine_grid->nodes->SetParent_CV(c, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 2, c); - fine_grid->nodes->SetParent_CV(d, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 3, d); - nodes->SetnChildren_CV(Index_CoarseCV, 4); - - reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - MGQueue_InnerCV.RemoveCV(c); - MGQueue_InnerCV.RemoveCV(d); - - Index_CoarseCV++; - line_processed[li1] = line_processed[li2] = 1; - for (auto other_li : line_ids) - if (other_li != li1 && other_li != li2) line_processed[other_li] = 1; - any_work = true; + const auto& L2 = lines[li2_best]; + const auto c = L2[idx1], d = L2[idx2]; + + /*--- Skip if any node is already claimed ---*/ + if (fine_grid->nodes->GetAgglomerate(c) || fine_grid->nodes->GetAgglomerate(d)) continue; + if (reserved[c] || reserved[d]) continue; + + /*--- Geometrical quality check ---*/ + if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config) || + !GeometricalCheck(c, fine_grid, config) || !GeometricalCheck(d, fine_grid, config)) + continue; + + /*--- Guard against duplicate indices ---*/ + if (a == b || a == c || a == d || b == c || b == d || c == d) { + line_processed[li1] = line_processed[li2_best] = 1; + continue; + } + + /*--- Create 4-child coarse CV (isotropic agglomeration) ---*/ + fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 0, a); + fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 1, b); + fine_grid->nodes->SetParent_CV(c, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 2, c); + fine_grid->nodes->SetParent_CV(d, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 3, d); + nodes->SetnChildren_CV(Index_CoarseCV, 4); + + /*--- Debug output: show CV creation details ---*/ + if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { + const auto* coord_a = fine_grid->nodes->GetCoord(a); + const auto* coord_b = fine_grid->nodes->GetCoord(b); + cout << " CV " << Index_CoarseCV << " (ISO): nodes " << a << "+" << b << "+" << c << "+" << d + << " | lines[" << li1 << "][" << idx1 << "," << idx2 << "]+lines[" << li2_best << "][" << idx1 << "," << idx2 << "]" + << " | coord_a=(" << coord_a[0] << "," << coord_a[1] << ")" + << " coord_b=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; } + + reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; + MGQueue_InnerCV.RemoveCV(a); + MGQueue_InnerCV.RemoveCV(b); + MGQueue_InnerCV.RemoveCV(c); + MGQueue_InnerCV.RemoveCV(d); + + Index_CoarseCV++; + line_processed[li1] = line_processed[li2_best] = 1; + any_work = true; } } else { - /*--- ANISOTROPIC MODE: Pair nodes at SAME position on DIFFERENT lines ---*/ - for (auto& [parent, line_ids] : parent_to_lines) { - if (line_ids.size() < 2) continue; - - /*--- Pair consecutive lines at the same position ---*/ - for (size_t k = 0; k + 1 < line_ids.size(); k += 2) { - const auto li1 = line_ids[k]; - const auto li2 = line_ids[k + 1]; - if (line_processed[li1] || line_processed[li2]) continue; - - const auto& L1 = lines[li1]; - const auto& L2 = lines[li2]; - const auto pos = 1 + position_idx; - if (L1.size() <= pos || L2.size() <= pos) continue; - - const auto a = L1[pos]; - const auto b = L2[pos]; - - /*--- Skip if any node is already claimed ---*/ - if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b)) continue; - if (reserved[a] || reserved[b]) continue; - - /*--- Geometrical quality check ---*/ - if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config)) continue; - - /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - nodes->SetnChildren_CV(Index_CoarseCV, 2); - - reserved[a] = reserved[b] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - - Index_CoarseCV++; - line_processed[li1] = line_processed[li2] = 1; - any_work = true; + /*--- ANISOTROPIC MODE: Pair nodes at SAME position on DIFFERENT lines + Use spatial neighbor search to pair adjacent lines. ---*/ + for (auto li1 : active_lines) { + if (line_processed[li1]) continue; + + const auto& L1 = lines[li1]; + const auto pos = 1 + position_idx; + if (L1.size() <= pos) continue; + + const auto a = L1[pos]; + if (fine_grid->nodes->GetAgglomerate(a)) continue; + if (reserved[a]) continue; + if (!GeometricalCheck(a, fine_grid, config)) continue; + + /*--- Find nearest neighbor line by checking mesh neighbors of node 'a' ---*/ + unsigned long li2_best = std::numeric_limits::max(); + for (auto neighbor_point : fine_grid->nodes->GetPoints(a)) { + /*--- Check if this neighbor belongs to another unprocessed line at same position ---*/ + for (auto li2 : active_lines) { + if (li2 == li1 || line_processed[li2]) continue; + const auto& L2 = lines[li2]; + if (L2.size() <= pos) continue; + const auto b = L2[pos]; + if (b == neighbor_point) { + li2_best = li2; + break; + } + } + if (li2_best != std::numeric_limits::max()) break; + } + + if (li2_best == std::numeric_limits::max()) { + /*--- Debug: Line couldn't find a neighbor ---*/ + if (DEBUG_OUTPUT && position_idx < 3) { + cout << " Line " << li1 << " at pos=" << pos << " (node " << a << ") has NO neighbor line!" << endl; + } + continue; + } + + const auto& L2 = lines[li2_best]; + const auto b = L2[pos]; + + /*--- Skip if partner is already claimed ---*/ + if (fine_grid->nodes->GetAgglomerate(b)) continue; + if (reserved[b]) continue; + + /*--- Geometrical quality check ---*/ + if (!GeometricalCheck(b, fine_grid, config)) continue; + + /*--- Debug: Check line distance and neighbor relationships ---*/ + if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { + /*--- Measure distance between wall vertices of the two lines ---*/ + const auto wall_a = lines[li1][0]; + const auto wall_b = lines[li2_best][0]; + const auto* coord_wall_a = fine_grid->nodes->GetCoord(wall_a); + const auto* coord_wall_b = fine_grid->nodes->GetCoord(wall_b); + su2double wall_dist = sqrt(pow(coord_wall_a[0] - coord_wall_b[0], 2) + + pow(coord_wall_a[1] - coord_wall_b[1], 2)); + + /*--- Check if wall vertices are neighbors ---*/ + bool walls_are_neighbors = false; + for (auto neighbor : fine_grid->nodes->GetPoints(wall_a)) { + if (neighbor == wall_b) { walls_are_neighbors = true; break; } + } + + cout << " Pairing lines " << li1 << " + " << li2_best << " at pos=" << pos + << " | wall_dist=" << wall_dist << " | walls_neighbors=" << (walls_are_neighbors ? "YES" : "NO") << endl; } + + /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ + fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 0, a); + fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, 1, b); + nodes->SetnChildren_CV(Index_CoarseCV, 2); + + /*--- Debug output: show CV creation details ---*/ + if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { + const auto* coord_a = fine_grid->nodes->GetCoord(a); + const auto* coord_b = fine_grid->nodes->GetCoord(b); + su2double dist = sqrt(pow(coord_a[0] - coord_b[0], 2) + pow(coord_a[1] - coord_b[1], 2)); + bool are_neighbors = false; + for (auto neighbor : fine_grid->nodes->GetPoints(a)) { + if (neighbor == b) { are_neighbors = true; break; } + } + cout << " CV " << Index_CoarseCV << " (ANISO): nodes " << a << "+" << b + << " | lines[" << li1 << "][" << pos << "]+lines[" << li2_best << "][" << pos << "]" + << " | dist=" << dist << " | neighbors=" << (are_neighbors ? "YES" : "NO") + << " | coords A=(" << coord_a[0] << "," << coord_a[1] << ")" + << " B=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; + } + + reserved[a] = reserved[b] = 1; + MGQueue_InnerCV.RemoveCV(a); + MGQueue_InnerCV.RemoveCV(b); + + Index_CoarseCV++; + line_processed[li1] = line_processed[li2_best] = 1; + any_work = true; } } @@ -1544,4 +1738,62 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } if (!any_more) break; } + + /*--- Count how many CVs and nodes were created ---*/ + const auto nCVs_created = Index_CoarseCV - starting_Index_CoarseCV; + unsigned long nNodes_claimed = 0; + unsigned long nNodes_on_lines = 0; + unsigned long nNodes_unpaired = 0; + + for (const auto& L : lines) { + for (size_t i = 1; i < L.size(); ++i) { // Skip wall node at [0] + nNodes_on_lines++; + if (!reserved[L[i]]) nNodes_unpaired++; + } + } + + for (unsigned long i = 0; i < nPointFine; ++i) { + if (reserved[i]) nNodes_claimed++; + } + + if (rank == MASTER_NODE) { + cout << " Created " << nCVs_created << " coarse CVs from " << nNodes_claimed << " fine nodes." << endl; + cout << " Nodes on implicit lines: " << nNodes_on_lines << " (paired=" << (nNodes_on_lines - nNodes_unpaired) + << ", unpaired=" << nNodes_unpaired << ")" << endl; + + if (nNodes_unpaired > 0) { + cout << " WARNING: " << nNodes_unpaired << " nodes on implicit lines were left unpaired!" << endl; + cout << " These will be processed by domain agglomeration (may create wrong orientation)." << endl; + + /*--- Show first few unpaired nodes ---*/ + unsigned long count = 0; + for (size_t li = 0; li < lines.size() && count < 10; ++li) { + const auto& L = lines[li]; + for (size_t i = 1; i < L.size() && count < 10; ++i) { + if (!reserved[L[i]]) { + cout << " Unpaired: line " << li << " node " << L[i] << " at position " << i << endl; + count++; + } + } + } + } + if (ISOTROPIC) { + cout << " Expected ratio: ~4 nodes per CV (actual: " << std::setprecision(2) << std::fixed + << (nCVs_created > 0 ? su2double(nNodes_claimed) / su2double(nCVs_created) : 0.0) << ")" << endl; + } else { + cout << " Expected ratio: ~2 nodes per CV (actual: " << std::setprecision(2) << std::fixed + << (nCVs_created > 0 ? su2double(nNodes_claimed) / su2double(nCVs_created) : 0.0) << ")" << endl; + } + } + + /*--- Verify all claimed nodes are properly marked as agglomerated ---*/ + unsigned long mismatches = 0; + for (unsigned long i = 0; i < nPointFine; ++i) { + if (reserved[i] && !fine_grid->nodes->GetAgglomerate(i)) { + mismatches++; + } + } + if (mismatches > 0 && rank == MASTER_NODE) { + cout << " WARNING: " << mismatches << " nodes marked as reserved but not agglomerated!" << endl; + } } diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 1c0b35ad53d..cb19c8db88d 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -154,10 +154,11 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, /*--- Full MG: advance to the next finer grid after a fixed number of * outer iterations on the current coarsest active level. - * We use 100 iterations per level (nMGLevels levels total) ---*/ + * The number of iterations per level is controlled by MG_STARTUP_ITER config option. ---*/ + const unsigned long startup_iter = config[iZone]->GetMGOptions().MG_Startup_Iter; const bool Convergence_FullMG = FullMG && (FinestMesh != MESH_0) && - (config[iZone]->GetInnerIter() % 100 == 99); + (config[iZone]->GetInnerIter() % startup_iter == startup_iter - 1); if (!config[iZone]->GetRestart() && FullMG && direct && ( Convergence_FullMG && (FinestMesh != MESH_0 ))) { @@ -197,10 +198,10 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, passivedouble CFL_local = cfl_base; for (unsigned short iMesh = FinestMesh; iMesh < nMGLevels; ++iMesh) { const unsigned short lvl = iMesh + 1; - /*--- Use per-level scaling factor; clamp to (0,1] to prevent coarse CFL from - * exceeding the fine CFL. Index into cflScaling is iMesh (0-based transition). ---*/ + /*--- Use per-level scaling factor to increase coarse CFL (allows values > 1.0). + * Index into cflScaling is iMesh (0-based transition). ---*/ const passivedouble scale = (iMesh < cflScaling.size()) - ? max(passivedouble{1e-6}, min(passivedouble{1.0}, SU2_TYPE::GetValue(cflScaling[iMesh]))) + ? max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[iMesh])) : passivedouble{0.25}; CFL_local *= scale; config[iZone]->SetCFL(lvl, CFL_local); diff --git a/config_template.cfg b/config_template.cfg index 9e2fe09703d..614338c5ba8 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1724,6 +1724,10 @@ MG_IMPLICIT_LINES_MAX_LENGTH= 20 % Anisotropic (NO): Pair cells normal to wall (2 cells per coarse CV, reduces mesh ~2x) % Isotropic (YES): Pair cells in all directions (4 cells per coarse CV, reduces mesh ~4x) MG_IMPLICIT_LINES_ISOTROPIC= NO +% +% Number of iterations on coarsest mesh during Full Multigrid (FMG) startup phase. +% After this many iterations, solution is prolongated to finer mesh (default 100). +MG_STARTUP_ITER= 100 % -------------------------- MESH SMOOTHING -----------------------------% % From 3b9e9dfbbb5e12d1196b0425c052cbf9b2f2bd8c Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sat, 1 Aug 2026 16:45:31 +0200 Subject: [PATCH 03/54] fix multigrid turbulence --- Common/include/option_structure.hpp | 1 + Common/src/CConfig.cpp | 2 + .../integration/CMultiGridIntegration.hpp | 20 +++ .../src/integration/CMultiGridIntegration.cpp | 131 +++++++++++++++++- SU2_CFD/src/iteration/CFluidIteration.cpp | 15 +- SU2_CFD/src/solvers/CSolverFactory.cpp | 2 +- SU2_CFD/src/solvers/CTurbSASolver.cpp | 2 +- SU2_CFD/src/solvers/CTurbSSTSolver.cpp | 2 +- SU2_CFD/src/variables/CTurbVariable.cpp | 12 ++ config_template.cfg | 4 + 10 files changed, 183 insertions(+), 8 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index e277e514332..14e09e9158f 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1128,6 +1128,7 @@ struct CMGOptions { unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ + bool TurbMG{false}; /*!< \brief Run turbulence equations through a FAS MG V-cycle instead of single-grid. */ }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 2d68a229b27..208767d6db5 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2073,6 +2073,8 @@ void CConfig::SetConfig_Options() { addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); + /*!\brief MG_TURB\n DESCRIPTION: Run turbulence equations through a FAS Multigrid V-cycle instead of single-grid. DEFAULT: NO \ingroup Config*/ + addBoolOption("MG_TURB", MGOptions.TurbMG, false); /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 0672d1ae5ff..37f68c33f8f 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -248,6 +248,26 @@ class CMultiGridIntegration final : public CIntegration { passivedouble lastRMS[2], char& exitReason, passivedouble& worstStepRatio, unsigned short& worstStep); + /*! + * \brief Restrict turbulent eddy viscosity from fine to coarser mesh levels. + * + * After a turbulence FAS V-cycle completes, this function volume-weights restricts + * mu_t from the finest mesh down to all coarser levels. The flow solver on the next + * outer iteration uses these restricted mu_t values at every coarse level for the + * eddy-viscosity coupling. This ensures consistency between flow and turbulence + * solutions across the multigrid hierarchy. + * + * \param[in] geometry - Geometry hierarchy for one zone/instance (all levels). + * \param[in] solver - Solver hierarchy for one zone/instance (all levels). + * \param[in] config - Problem configuration. + * \param[in] FinestMesh - Current finest active mesh index. + * \param[in] nMGLevels - Total number of MG levels. + */ + void RestrictTurbEddyViscToCoarseLevels(CGeometry** geometry, CSolver*** solver, + CConfig* config, + unsigned short FinestMesh, + unsigned short nMGLevels); + static constexpr int MAX_MG_LEVELS = 10; /*--- Early-exit smoothing state (shared across OMP threads via master write + barrier). ---*/ diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index cb19c8db88d..6ebbfc9eeb5 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -28,6 +28,11 @@ #include "../../include/integration/CMultiGridIntegration.hpp" #include "../../../Common/include/parallelization/omp_structure.hpp" #include "../../../Common/include/toolboxes/printing_toolbox.hpp" +#include +#include +#include + +using namespace std; namespace { @@ -160,7 +165,8 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, FullMG && (FinestMesh != MESH_0) && (config[iZone]->GetInnerIter() % startup_iter == startup_iter - 1); - if (!config[iZone]->GetRestart() && FullMG && direct && ( Convergence_FullMG && (FinestMesh != MESH_0 ))) { + if (!config[iZone]->GetRestart() && FullMG && direct && ( Convergence_FullMG && (FinestMesh != MESH_0 )) && + RunTime_EqSystem == RUNTIME_FLOW_SYS) { SetProlongated_Solution(RunTime_EqSystem, solver_container[iZone][iInst][FinestMesh-1][Solver_Position], @@ -169,6 +175,46 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, geometry[iZone][iInst][FinestMesh], config[iZone]); + /*--- Prolongate scalar solutions to the new finest mesh. + * All scalar solvers (turb, species, transition) run via SingleGrid_Iteration on + * GetFinestMesh(). Only turbulence additionally restricts its field downward to + * coarser meshes; no scalar ever propagates upward to finer meshes. Consequently + * meshes finer than FinestMesh hold their iter-0 startup values for the entire + * warmup phase. When FinestMesh is decremented these stale fields cause a large + * transient (e.g. +3 decade regression in rms[nu]). Prolongating here mirrors + * what SetProlongated_Solution does for the flow and eliminates the regression. ---*/ + if (config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { + SetProlongated_Solution(RUNTIME_TURB_SYS, + solver_container[iZone][iInst][FinestMesh-1][TURB_SOL], + solver_container[iZone][iInst][FinestMesh][TURB_SOL], + geometry[iZone][iInst][FinestMesh-1], + geometry[iZone][iInst][FinestMesh], + config[iZone]); + /*--- Recompute mu_t on the new finest mesh from the prolongated nu_tilde/k/omega. ---*/ + solver_container[iZone][iInst][FinestMesh-1][TURB_SOL]->Postprocessing( + geometry[iZone][iInst][FinestMesh-1], + solver_container[iZone][iInst][FinestMesh-1], + config[iZone], FinestMesh-1); + } + + if (config[iZone]->GetKind_Trans_Model() == TURB_TRANS_MODEL::LM) { + SetProlongated_Solution(RUNTIME_TRANS_SYS, + solver_container[iZone][iInst][FinestMesh-1][TRANS_SOL], + solver_container[iZone][iInst][FinestMesh][TRANS_SOL], + geometry[iZone][iInst][FinestMesh-1], + geometry[iZone][iInst][FinestMesh], + config[iZone]); + } + + if (config[iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { + SetProlongated_Solution(RUNTIME_SPECIES_SYS, + solver_container[iZone][iInst][FinestMesh-1][SPECIES_SOL], + solver_container[iZone][iInst][FinestMesh][SPECIES_SOL], + geometry[iZone][iInst][FinestMesh-1], + geometry[iZone][iInst][FinestMesh], + config[iZone]); + } + SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) } @@ -176,11 +222,45 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, FinestMesh = config[iZone]->GetFinestMesh(); + /*--- For turbulence MG: before descending to coarse levels, ensure mu_t is computed + * at the finest level and restricted to all coarser levels. This prevents inf + * residuals from coarse-level turbulence solves using stale/uninitialized mu_t. ---*/ + if (RunTime_EqSystem == RUNTIME_TURB_SYS && + config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { + + solver_container[iZone][iInst][FinestMesh][TURB_SOL]->Postprocessing( + geometry[iZone][iInst][FinestMesh], + solver_container[iZone][iInst][FinestMesh], + config[iZone], FinestMesh); + + RestrictTurbEddyViscToCoarseLevels(geometry[iZone][iInst], + solver_container[iZone][iInst], + config[iZone], FinestMesh, + config[iZone]->GetnMGLevels()); + } + /*--- Perform the Full Approximation Scheme multigrid ---*/ MultiGrid_Cycle(geometry, solver_container, numerics_container, config, FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); + /*--- After a turb FAS V-cycle: recompute mu_t at the finest active level from the updated + * nu_hat/k/omega and restrict it to all coarser levels. The flow FAS on the NEXT outer + * iteration uses these mu_t values at every coarse level for the eddy-viscosity coupling. + * (Postprocessing was already called on FinestMesh inside the last PreSmoothing step of + * MultiGrid_Cycle; we call it once more to be safe after the V-cycle correction is applied.) ---*/ + if (RunTime_EqSystem == RUNTIME_TURB_SYS && + config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { + solver_container[iZone][iInst][FinestMesh][TURB_SOL]->Postprocessing( + geometry[iZone][iInst][FinestMesh], + solver_container[iZone][iInst][FinestMesh], + config[iZone], FinestMesh); + RestrictTurbEddyViscToCoarseLevels(geometry[iZone][iInst], + solver_container[iZone][iInst], + config[iZone], FinestMesh, + config[iZone]->GetnMGLevels()); + } + /*--- Adapt coarse-grid CFL once per cycle using smoothing residuals gathered during the cycle. ---*/ const unsigned short nMGLevels = config[iZone]->GetnMGLevels(); BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS @@ -284,8 +364,13 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, return ss.str(); }; + const string eqName = (RunTime_EqSystem == RUNTIME_FLOW_SYS) ? "Flow" : + (RunTime_EqSystem == RUNTIME_TURB_SYS) ? "Turb" : + (RunTime_EqSystem == RUNTIME_SPECIES_SYS) ? "Species" : + (RunTime_EqSystem == RUNTIME_TRANS_SYS) ? "Trans" : "Other"; + PrintingToolbox::CTablePrinter table(&std::cout); - table.AddColumn("Smoother", 13); + table.AddColumn("Smoother [" + eqName + "]", 13 + 7); for (unsigned short i = 0; i <= nMGLevels; ++i) table.AddColumn("Level " + std::to_string(i), 38); table.PrintHeader(); @@ -412,6 +497,15 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, solver_coarse->Preprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem, false); + /*--- For turbulence: ensure flow primitives (density, laminar viscosity) are updated on the + * coarse level from the restricted conservative variables, THEN compute mu_t from the + * newly restricted turbulence variables. This ensures turbulence Postprocessing reads + * valid flow data and Space_Integration uses correct eddy viscosity. ---*/ + if (RunTime_EqSystem == RUNTIME_TURB_SYS && config->GetKind_Turb_Model() != TURB_MODEL::NONE) { + solver_container_coarse[FLOW_SOL]->Preprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1, NO_RK_ITER, RUNTIME_FLOW_SYS, false); + solver_coarse->Postprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1); + } + Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); /*--- Compute $P_(k+1) = I^(k+1)_k(r_k) - r_(k+1) ---*/ @@ -1095,3 +1189,36 @@ void CMultiGridIntegration::Adjoint_Setup(CGeometry ****geometry, CSolver *****s } } + +void CMultiGridIntegration::RestrictTurbEddyViscToCoarseLevels(CGeometry** geometry, CSolver*** solver, + CConfig* config, + unsigned short FinestMesh, + unsigned short nMGLevels) { + SU2_ZONE_SCOPED + + for (unsigned short iMesh = FinestMesh; iMesh < nMGLevels; iMesh++) { + + CGeometry* geo_fine = geometry[iMesh]; + CGeometry* geo_coarse = geometry[iMesh + 1]; + CSolver* sol_fine = solver[iMesh][TURB_SOL]; + CSolver* sol_coarse = solver[iMesh + 1][TURB_SOL]; + + /*--- Volume-weighted restriction of mu_t from fine to coarse. ---*/ + SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) + for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { + + const su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); + su2double EddyVisc = 0.0; + + for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { + auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); + su2double Area_Children = geo_fine->nodes->GetVolume(Point_Fine); + su2double mu_t = sol_fine->GetNodes()->GetmuT(Point_Fine); + EddyVisc += mu_t * Area_Children / Area_Parent; + } + + sol_coarse->GetNodes()->SetmuT(Point_Coarse, EddyVisc); + } + END_SU2_OMP_FOR + } +} diff --git a/SU2_CFD/src/iteration/CFluidIteration.cpp b/SU2_CFD/src/iteration/CFluidIteration.cpp index a79a9f1f8f4..bdfd71fb8fc 100644 --- a/SU2_CFD/src/iteration/CFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CFluidIteration.cpp @@ -82,7 +82,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe integration[val_iZone][val_iInst][FLOW_SOL]->MultiGrid_Iteration(geometry, solver, numerics, config, RUNTIME_FLOW_SYS, val_iZone, val_iInst); - /*--- If the flow integration is not fully coupled, run the various single grid integrations. ---*/ + /*--- If the flow integration is not fully coupled, run the various single/multi-grid integrations. ---*/ if (config[val_iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE && !frozen_visc) { @@ -95,14 +95,22 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe } /*--- Solve the turbulence model ---*/ + /*--- Use multigrid if MG_TURB is enabled, otherwise use single-grid. ---*/ config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_TURB_SYS); - integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, - RUNTIME_TURB_SYS, val_iZone, val_iInst); + + if (config[val_iZone]->GetMGOptions().TurbMG) { + integration[val_iZone][val_iInst][TURB_SOL]->MultiGrid_Iteration(geometry, solver, numerics, config, + RUNTIME_TURB_SYS, val_iZone, val_iInst); + } else { + integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, + RUNTIME_TURB_SYS, val_iZone, val_iInst); + } } if (config[val_iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_SPECIES_SYS); + /*--- Use multigrid if MG_SPECIES is enabled (future feature), otherwise use single-grid. ---*/ integration[val_iZone][val_iInst][SPECIES_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_SPECIES_SYS, val_iZone, val_iInst); @@ -119,6 +127,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe if (config[val_iZone]->GetWeakly_Coupled_Heat()) { config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_HEAT_SYS); + /*--- Use multigrid if MG_HEAT is enabled (future feature), otherwise use single-grid. ---*/ integration[val_iZone][val_iInst][HEAT_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_HEAT_SYS, val_iZone, val_iInst); } diff --git a/SU2_CFD/src/solvers/CSolverFactory.cpp b/SU2_CFD/src/solvers/CSolverFactory.cpp index ee798c1384b..00e7064d31c 100644 --- a/SU2_CFD/src/solvers/CSolverFactory.cpp +++ b/SU2_CFD/src/solvers/CSolverFactory.cpp @@ -313,7 +313,7 @@ CSolver* CSolverFactory::CreateSubSolver(SUB_SOLVER_TYPE kindSolver, CSolver **s case SUB_SOLVER_TYPE::TURB_SA: case SUB_SOLVER_TYPE::TURB_SST: genericSolver = CreateTurbSolver(kindTurbModel, solver, geometry, config, iMGLevel, false); - metaData.integrationType = INTEGRATION_TYPE::SINGLEGRID; + metaData.integrationType = config->GetMGOptions().TurbMG ? INTEGRATION_TYPE::MULTIGRID : INTEGRATION_TYPE::SINGLEGRID; break; case SUB_SOLVER_TYPE::TEMPLATE: genericSolver = new CTemplateSolver(geometry, config); diff --git a/SU2_CFD/src/solvers/CTurbSASolver.cpp b/SU2_CFD/src/solvers/CTurbSASolver.cpp index b58afe4c9b3..714d1972695 100644 --- a/SU2_CFD/src/solvers/CTurbSASolver.cpp +++ b/SU2_CFD/src/solvers/CTurbSASolver.cpp @@ -65,7 +65,7 @@ CTurbSASolver::CTurbSASolver(CGeometry *geometry, CConfig *config, unsigned shor /*--- Single grid simulation ---*/ - if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL) { + if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL || config->GetMGOptions().TurbMG) { /*--- Define some auxiliar vector related with the residual ---*/ diff --git a/SU2_CFD/src/solvers/CTurbSSTSolver.cpp b/SU2_CFD/src/solvers/CTurbSSTSolver.cpp index 579b5f3cf72..97b4ee1d78c 100644 --- a/SU2_CFD/src/solvers/CTurbSSTSolver.cpp +++ b/SU2_CFD/src/solvers/CTurbSSTSolver.cpp @@ -59,7 +59,7 @@ CTurbSSTSolver::CTurbSSTSolver(CGeometry *geometry, CConfig *config, unsigned sh /*--- Single grid simulation ---*/ - if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL) { + if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL || config->GetMGOptions().TurbMG) { /*--- Define some auxiliary vector related with the residual ---*/ diff --git a/SU2_CFD/src/variables/CTurbVariable.cpp b/SU2_CFD/src/variables/CTurbVariable.cpp index 139223055a4..62b87f9363d 100644 --- a/SU2_CFD/src/variables/CTurbVariable.cpp +++ b/SU2_CFD/src/variables/CTurbVariable.cpp @@ -35,6 +35,18 @@ CTurbVariable::CTurbVariable(unsigned long npoint, unsigned long ndim, unsigned turb_index.resize(nPoint) = su2double(1.0); intermittency.resize(nPoint) = su2double(1.0); + /*--- Allocate residual structures for multigrid (required for turbulence MG). ---*/ + Res_TruncError.resize(nPoint, nVar) = su2double(0.0); + + /*--- Allocate smoothing arrays if correction smoothing is enabled at any MG level. ---*/ + for (unsigned long iMesh = 0; iMesh <= config->GetnMGLevels(); iMesh++) { + if (config->GetMGOptions().MG_CorrecSmooth[iMesh] > 0) { + Residual_Sum.resize(nPoint, nVar); + Residual_Old.resize(nPoint, nVar); + break; + } + } + } void CTurbVariable::RegisterEddyViscosity(bool input) { diff --git a/config_template.cfg b/config_template.cfg index 45a822755c9..2c7d882c11f 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1732,6 +1732,10 @@ MG_IMPLICIT_LINES_ISOTROPIC= NO % Number of iterations on coarsest mesh during Full Multigrid (FMG) startup phase. % After this many iterations, solution is prolongated to finer mesh (default 100). MG_STARTUP_ITER= 100 +% +% Run turbulence equations through a FAS Multigrid V-cycle (YES, NO) +% When disabled, turbulence is solved with single-grid only (default NO). +MG_TURB= NO % -------------------------- MESH SMOOTHING -----------------------------% % From 7e1f670ab24b22f3b1e3a9d2cccce5d4ad531d39 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 2 Aug 2026 13:47:34 +0200 Subject: [PATCH 04/54] small flow solver update --- .../integration/CMultiGridIntegration.hpp | 26 +++ .../src/integration/CMultiGridIntegration.cpp | 180 ++++++++++++++++-- 2 files changed, 190 insertions(+), 16 deletions(-) diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 37f68c33f8f..1b5c7476bf6 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -81,6 +81,18 @@ class CMultiGridIntegration final : public CIntegration { void SetForcing_Term(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config, unsigned short iMesh); + /*! + * \brief Restrict the fine-grid residual defect to the coarse-grid FAS forcing term. + * \param[in] sol_fine - Pointer to the solution on the fine grid. + * \param[in] sol_coarse - Pointer to the solution on the coarse grid. + * \param[in] geo_fine - Geometrical definition of the fine grid. + * \param[in] geo_coarse - Geometrical definition of the coarse grid. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void RestrictResidualToCoarseGrid(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, + CGeometry *geo_coarse, CConfig *config, unsigned short iMesh); + /*! * \brief Add the truncation error to the residual. * \param[in] geometry - Geometrical definition of the problem. @@ -180,6 +192,20 @@ class CMultiGridIntegration final : public CIntegration { void GetProlongated_Correction(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config); + /*! + * \brief Prolongate the coarse-grid state correction back to the fine-grid residual correction. + * \param[in] RunTime_EqSystem - System of equations which is going to be solved. + * \param[in] sol_fine - Pointer to the solution on the fine grid. + * \param[in] sol_coarse - Pointer to the solution on the coarse grid. + * \param[in] geo_fine - Geometrical definition of the fine grid. + * \param[in] geo_coarse - Geometrical definition of the coarse grid. + * \param[in] config - Definition of the particular problem. + * \param[in] iMesh - Index of the mesh in multigrid computations. + */ + void ProlongateCorrectionToFineGrid(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, + CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config, + unsigned short iMesh); + /*! * \brief Do an implicit smoothing of the prolongated correction. * \param[in] RunTime_EqSystem - System of equations which is going to be solved. diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 6ebbfc9eeb5..91097a635d2 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -460,7 +460,8 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, PreSmoothing(RunTime_EqSystem, geometry, solver_container, config_container, solver_fine, numerics_fine, geometry_fine, solver_container_fine, config, iMesh, iZone, iRKLimit); - /*--- Compute Forcing Term $P_(k+1) = I^(k+1)_k(P_k+F_k(u_k))-F_(k+1)(I^(k+1)_k u_k)$ and update solution for multigrid ---*/ + /*--- Assemble the coarse-grid FAS defect term by restricting the fine-grid residual defect, + * solving the coarse-grid state, and prolongating only the state correction back to the fine grid. ---*/ if ( iMesh < config->GetnMGLevels() ) { @@ -483,15 +484,16 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, Space_Integration(geometry_fine, solver_container_fine, numerics_fine, config, iMesh, NO_RK_ITER, RunTime_EqSystem); - /*--- LinSysRes = R(u_N) here, before tau is added by SetResidual_Term. ---*/ + /*--- LinSysRes = R(u_N) here, before the fine-grid defect term is assembled. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { lastPreSmoothRMS[iMesh][1] = ComputeLinSysResRMS(solver_fine); } END_SU2_OMP_SAFE_GLOBAL_ACCESS + /*--- Assemble the fine-grid defect term that will be restricted to the coarse-grid FAS problem. ---*/ SetResidual_Term(geometry_fine, solver_fine); - /*--- Compute $r_(k+1) = F_(k+1)(I^(k+1)_k u_k)$ ---*/ + /*--- Restrict the fine-grid state to the coarse grid and initialize the coarse-grid state. ---*/ SetRestricted_Solution(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); @@ -508,9 +510,12 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); - /*--- Compute $P_(k+1) = I^(k+1)_k(r_k) - r_(k+1) ---*/ - - SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); + /*--- Restrict the fine-grid residual defect to the coarse-grid FAS forcing term. ---*/ + if (RunTime_EqSystem == RUNTIME_FLOW_SYS) { + RestrictResidualToCoarseGrid(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); + } else { + SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); + } /*--- Restore the time integration settings. ---*/ @@ -532,9 +537,12 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, iMesh+1, nextRecurseParam, RunTime_EqSystem, iZone, iInst); } - /*--- Compute prolongated solution, and smooth the correction $u^(new)_k = u_k + Smooth(I^k_(k+1)(u_(k+1)-I^(k+1)_k u_k))$ ---*/ - - GetProlongated_Correction(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); + /*--- Compute the coarse-grid state correction and prolongate it back to the fine grid. ---*/ + if (RunTime_EqSystem == RUNTIME_FLOW_SYS) { + ProlongateCorrectionToFineGrid(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh); + } else { + GetProlongated_Correction(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); + } const auto& mgOpts = config->GetMGOptions(); SmoothProlongated_Correction(RunTime_EqSystem, solver_fine, geometry_fine, mgOpts.MG_CorrecSmooth[iMesh], mgOpts.MG_Smooth_Coeff, config, iMesh); @@ -823,6 +831,8 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ if (val_nSmooth == 0) return; const unsigned short nVar = solver->GetnVar(); + const bool use_conservative_damping = (nVar <= 2); + const su2double turbulence_base_damping = 0.50; SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); iPoint++) { @@ -862,8 +872,11 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ const auto* Residual_Sum = solver->GetNodes()->GetResidual_Sum(iPoint); const auto* Residual_Old = solver->GetNodes()->GetResidual_Old(iPoint); - for (auto iVar = 0u; iVar < nVar; iVar++) - solver->LinSysRes(iPoint,iVar) = (Residual_Old[iVar] + val_smooth_coeff*Residual_Sum[iVar])*factor; + for (auto iVar = 0u; iVar < nVar; iVar++) { + su2double smoothed = (Residual_Old[iVar] + val_smooth_coeff*Residual_Sum[iVar])*factor; + if (use_conservative_damping) smoothed *= turbulence_base_damping; + solver->LinSysRes(iPoint,iVar) = smoothed; + } } END_SU2_OMP_FOR @@ -890,6 +903,11 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ if (config->GetMGOptions().MG_Smooth_Output) { const su2double res = sqrt(solver->LinSysRes.squaredNorm() / (nVar * geometry->GetGlobal_nPointDomain())); SU2_OMP_SAFE_GLOBAL_ACCESS(lastCorrecSmoothRMS[iMesh][1] = SU2_TYPE::GetValue(res);) + + if (SU2_MPI::GetRank() == MASTER_NODE && use_conservative_damping) { + cout << "[MG CORR-SMOOTH] turbulence nSmooth=" << val_nSmooth + << " norm=" << res << "\n"; + } } } @@ -898,25 +916,129 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet SU2_ZONE_SCOPED const unsigned short nVar = sol_fine->GetnVar(); + const bool use_conservative_damping = (nVar <= 2); + const su2double base_damping = use_conservative_damping ? 0.50 : 1.0; + const su2double wall_damping = use_conservative_damping ? 0.25 : 1.0; /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); + vector isWall(geo_fine->GetnPoint(), false); + for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) + if (config->GetViscous_Wall(iMarker)) + for (auto iVertex = 0ul; iVertex < geo_fine->nVertex[iMarker]; iVertex++) + isWall[geo_fine->vertex[iMarker][iVertex]->GetNode()] = true; + SU2_OMP_FOR_STAT(roundUpDiv(geo_fine->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Fine = 0ul; Point_Fine < geo_fine->GetnPointDomain(); Point_Fine++) { auto* Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); auto* Solution_Fine = sol_fine->GetNodes()->GetSolution(Point_Fine); + + su2double residualMag = 0.0; + su2double correctionMag = 0.0; for (auto iVar = 0u; iVar < nVar; iVar++) { /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ - if (Residual_Fine[iVar] != Residual_Fine[iVar]) + if (Residual_Fine[iVar] != Residual_Fine[iVar]) { Residual_Fine[iVar] = 0.0; + } + const su2double corr = factor * Residual_Fine[iVar]; + residualMag = max(residualMag, fabs(Residual_Fine[iVar])); + correctionMag = max(correctionMag, fabs(corr)); + } + + su2double correctionScale = 1.0; + if (residualMag > 1e-30 && correctionMag > 1e-30) { + const su2double ratio = correctionMag / residualMag; + if (ratio > 2.0) { + correctionScale = 2.0 / ratio; + } + } + + const su2double localDamping = use_conservative_damping ? (isWall[Point_Fine] ? wall_damping : base_damping) : 1.0; + for (auto iVar = 0u; iVar < nVar; iVar++) { su2double correction = factor * Residual_Fine[iVar]; + correction *= localDamping; + correction *= correctionScale; + + if (!std::isfinite(correction)) { + correction = 0.0; + } + Solution_Fine[iVar] += correction; } } END_SU2_OMP_FOR + /*--- DIAGNOSTIC: log the max applied correction (factor * LinSysRes) at fine-grid wall points + * vs interior. ---*/ + if (config->GetMGOptions().MG_Smooth_Output && SU2_MPI::GetRank() == MASTER_NODE) { + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS + { + if (nVar > 2) { + su2double maxWall0 = 0.0, maxWallN = 0.0, maxWallMom = 0.0; + su2double maxInter0 = 0.0, maxInterN = 0.0, maxInterMom = 0.0; + su2double maxWallApply0 = 0.0, maxWallApplyN = 0.0, maxWallApplyMom = 0.0; + su2double maxInterApply0 = 0.0, maxInterApplyN = 0.0, maxInterApplyMom = 0.0; + + for (auto iPoint = 0ul; iPoint < geo_fine->GetnPointDomain(); iPoint++) { + const auto* corr = sol_fine->LinSysRes.GetBlock(iPoint); + const su2double localDamping = isWall[iPoint] ? wall_damping : base_damping; + const su2double applied0 = fabs(localDamping * factor * corr[0]); + const su2double appliedN = fabs(localDamping * factor * corr[nVar-1]); + su2double appliedMom = 0.0; + for (auto iVar = 1u; iVar < static_cast(nVar-1); iVar++) { + appliedMom = max(appliedMom, fabs(localDamping * factor * corr[iVar])); + } + + if (isWall[iPoint]) { + maxWall0 = max(maxWall0, fabs(factor * corr[0])); + maxWallN = max(maxWallN, fabs(factor * corr[nVar-1])); + maxWallMom = max(maxWallMom, fabs(factor * corr[0])); + maxWallApply0 = max(maxWallApply0, applied0); + maxWallApplyN = max(maxWallApplyN, appliedN); + maxWallApplyMom = max(maxWallApplyMom, appliedMom); + } else { + maxInter0 = max(maxInter0, fabs(factor * corr[0])); + maxInterN = max(maxInterN, fabs(factor * corr[nVar-1])); + maxInterMom = max(maxInterMom, fabs(factor * corr[0])); + maxInterApply0 = max(maxInterApply0, applied0); + maxInterApplyN = max(maxInterApplyN, appliedN); + maxInterApplyMom = max(maxInterApplyMom, appliedMom); + } + } + auto ratio = [](su2double w, su2double i) { return (i > 1e-30) ? w/i : 0.0; }; + cout << "[MG APPL wall/inter] rho=" << ratio(maxWallApply0, maxInterApply0) + << " mom=" << ratio(maxWallApplyMom, maxInterApplyMom) + << " E=" << ratio(maxWallApplyN, maxInterApplyN) + << " (raw wall/inter: rho=" << maxWall0 << "/" << maxInter0 + << ", E=" << maxWallN << "/" << maxInterN + << "; applied wall/inter: rho=" << maxWallApply0 << "/" << maxInterApply0 + << ", E=" << maxWallApplyN << "/" << maxInterApplyN << ")\n"; + } else { + su2double maxWall = 0.0, maxInter = 0.0; + su2double maxWallApply = 0.0, maxInterApply = 0.0; + for (auto iPoint = 0ul; iPoint < geo_fine->GetnPointDomain(); iPoint++) { + const auto* corr = sol_fine->LinSysRes.GetBlock(iPoint); + const su2double localDamping = isWall[iPoint] ? wall_damping : base_damping; + const su2double mag = fabs(factor * corr[0]); + const su2double appliedMag = fabs(localDamping * factor * corr[0]); + if (isWall[iPoint]) { + maxWall = max(maxWall, mag); + maxWallApply = max(maxWallApply, appliedMag); + } else { + maxInter = max(maxInter, mag); + maxInterApply = max(maxInterApply, appliedMag); + } + } + cout << "[MG TURB APPLY] damp(wall/interior)= " << wall_damping << "/" << base_damping + << " raw max(wall/interior)= " << maxWall << "/" << maxInter + << " applied max(wall/interior)= " << maxWallApply << "/" << maxInterApply << "\n"; + } + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS + } + /*--- MPI the new interpolated solution ---*/ sol_fine->InitiateComms(geo_fine, config, MPI_QUANTITIES::SOLUTION); @@ -947,21 +1069,21 @@ void CMultiGridIntegration::SetForcing_Term(CSolver *sol_fine, CSolver *sol_coar const unsigned short nVar = sol_coarse->GetnVar(); const su2double factor = config->GetDamp_Res_Restric(); - su2activevector Residual(nVar); + su2activevector RestrictedDefect(nVar); SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { sol_coarse->GetNodes()->SetRes_TruncErrorZero(Point_Coarse); - Residual = su2double(0); + RestrictedDefect = su2double(0); for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); for (auto iVar = 0u; iVar < nVar; iVar++) - Residual[iVar] += factor * Residual_Fine[iVar]; + RestrictedDefect[iVar] += factor * Residual_Fine[iVar]; } - sol_coarse->GetNodes()->AddRes_TruncError(Point_Coarse, Residual.data()); + sol_coarse->GetNodes()->AddRes_TruncError(Point_Coarse, RestrictedDefect.data()); } END_SU2_OMP_FOR @@ -996,6 +1118,32 @@ void CMultiGridIntegration::SetResidual_Term(CGeometry *geometry, CSolver *solve } +void CMultiGridIntegration::RestrictResidualToCoarseGrid(CSolver *sol_fine, CSolver *sol_coarse, + CGeometry *geo_fine, CGeometry *geo_coarse, + CConfig *config, unsigned short iMesh) { + SU2_ZONE_SCOPED + + /*--- This is the standard FAS restriction step: the fine-grid defect is passed to the + * coarse-grid problem as a forcing term. The existing SetForcing_Term routine already + * implements the conservative volume-weighted transfer and the damping factor in the + * same way the original MG cycle expects. ---*/ + SetForcing_Term(sol_fine, sol_coarse, geo_fine, geo_coarse, config, iMesh); +} + +void CMultiGridIntegration::ProlongateCorrectionToFineGrid(unsigned short RunTime_EqSystem, CSolver *sol_fine, + CSolver *sol_coarse, CGeometry *geo_fine, + CGeometry *geo_coarse, CConfig *config, + unsigned short iMesh) { + SU2_ZONE_SCOPED + + /*--- This is the standard FAS prolongation step: build the coarse-grid state correction, + * then transfer that correction to the fine-grid residual correction. The original + * GetProlongated_Correction routine already performs this transfer in the correct form; + * the additional scaling here would be equivalent to changing the correction operator. + * Keep the transfer operator unchanged and let the existing damping path control the size. ---*/ + GetProlongated_Correction(RunTime_EqSystem, sol_fine, sol_coarse, geo_fine, geo_coarse, config); +} + void CMultiGridIntegration::SetRestricted_Solution(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config) { SU2_ZONE_SCOPED From 33f2eaf14d55ffea6e3ace0204ba006b73629a95 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 2 Aug 2026 14:45:17 +0200 Subject: [PATCH 05/54] small flow solver update --- .../src/integration/CMultiGridIntegration.cpp | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 91097a635d2..12997293c87 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -58,6 +58,15 @@ static su2double applyGlobalTrend(su2double factor, passivedouble crossCycleRati return max(su2double{CLAMP_MIN}, min(su2double{CLAMP_MAX}, factor)); } +static su2double GetMGLevelCorrectionScale(unsigned short iMesh) { + switch (iMesh) { + case 0: return 1.00; + case 1: return 0.75; + case 2: return 0.50; + default: return 0.35; + } +} + inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { passivedouble result = 0; for (unsigned short iVar = 0; iVar < solver->GetnVar(); ++iVar) { @@ -917,8 +926,9 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet const unsigned short nVar = sol_fine->GetnVar(); const bool use_conservative_damping = (nVar <= 2); - const su2double base_damping = use_conservative_damping ? 0.50 : 1.0; - const su2double wall_damping = use_conservative_damping ? 0.25 : 1.0; + const su2double levelScale = GetMGLevelCorrectionScale(iMesh); + const su2double base_damping = use_conservative_damping ? max(su2double{0.15}, 0.50 * levelScale) : 1.0; + const su2double wall_damping = use_conservative_damping ? max(su2double{0.10}, 0.25 * levelScale) : 1.0; /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); @@ -948,10 +958,11 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet } su2double correctionScale = 1.0; + constexpr su2double maxAllowedRatio = 1.25; if (residualMag > 1e-30 && correctionMag > 1e-30) { const su2double ratio = correctionMag / residualMag; - if (ratio > 2.0) { - correctionScale = 2.0 / ratio; + if (ratio > maxAllowedRatio) { + correctionScale = maxAllowedRatio / ratio; } } @@ -1008,13 +1019,14 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet } } auto ratio = [](su2double w, su2double i) { return (i > 1e-30) ? w/i : 0.0; }; - cout << "[MG APPL wall/inter] rho=" << ratio(maxWallApply0, maxInterApply0) + cout << "[MG APPL L" << iMesh << " wall/inter] rho=" << ratio(maxWallApply0, maxInterApply0) << " mom=" << ratio(maxWallApplyMom, maxInterApplyMom) << " E=" << ratio(maxWallApplyN, maxInterApplyN) << " (raw wall/inter: rho=" << maxWall0 << "/" << maxInter0 << ", E=" << maxWallN << "/" << maxInterN << "; applied wall/inter: rho=" << maxWallApply0 << "/" << maxInterApply0 - << ", E=" << maxWallApplyN << "/" << maxInterApplyN << ")\n"; + << ", E=" << maxWallApplyN << "/" << maxInterApplyN + << "; levelScale=" << levelScale << ", damp(wall/inter)=" << wall_damping << "/" << base_damping << ")\n"; } else { su2double maxWall = 0.0, maxInter = 0.0; su2double maxWallApply = 0.0, maxInterApply = 0.0; @@ -1031,9 +1043,10 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet maxInterApply = max(maxInterApply, appliedMag); } } - cout << "[MG TURB APPLY] damp(wall/interior)= " << wall_damping << "/" << base_damping + cout << "[MG TURB APPLY L" << iMesh << "] damp(wall/interior)= " << wall_damping << "/" << base_damping << " raw max(wall/interior)= " << maxWall << "/" << maxInter - << " applied max(wall/interior)= " << maxWallApply << "/" << maxInterApply << "\n"; + << " applied max(wall/interior)= " << maxWallApply << "/" << maxInterApply + << " levelScale=" << levelScale << "\n"; } } END_SU2_OMP_SAFE_GLOBAL_ACCESS From 3e0d528e4d2e0dd1896b400a33a203dd4afddd21 Mon Sep 17 00:00:00 2001 From: Nijso Date: Tue, 4 Aug 2026 21:28:47 +0200 Subject: [PATCH 06/54] Apply suggestions from code review remove multigrid turbulence Co-authored-by: Nijso --- Common/include/option_structure.hpp | 1 - Common/src/CConfig.cpp | 2 - .../integration/CMultiGridIntegration.hpp | 46 ----- .../src/integration/CMultiGridIntegration.cpp | 163 +----------------- SU2_CFD/src/iteration/CFluidIteration.cpp | 10 +- SU2_CFD/src/solvers/CSolverFactory.cpp | 2 +- SU2_CFD/src/solvers/CTurbSASolver.cpp | 2 +- SU2_CFD/src/solvers/CTurbSSTSolver.cpp | 2 +- SU2_CFD/src/variables/CTurbVariable.cpp | 12 -- config_template.cfg | 4 - 10 files changed, 7 insertions(+), 237 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 14e09e9158f..e277e514332 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1128,7 +1128,6 @@ struct CMGOptions { unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ - bool TurbMG{false}; /*!< \brief Run turbulence equations through a FAS MG V-cycle instead of single-grid. */ }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 208767d6db5..2d68a229b27 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2073,8 +2073,6 @@ void CConfig::SetConfig_Options() { addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); - /*!\brief MG_TURB\n DESCRIPTION: Run turbulence equations through a FAS Multigrid V-cycle instead of single-grid. DEFAULT: NO \ingroup Config*/ - addBoolOption("MG_TURB", MGOptions.TurbMG, false); /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 1b5c7476bf6..0672d1ae5ff 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -81,18 +81,6 @@ class CMultiGridIntegration final : public CIntegration { void SetForcing_Term(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config, unsigned short iMesh); - /*! - * \brief Restrict the fine-grid residual defect to the coarse-grid FAS forcing term. - * \param[in] sol_fine - Pointer to the solution on the fine grid. - * \param[in] sol_coarse - Pointer to the solution on the coarse grid. - * \param[in] geo_fine - Geometrical definition of the fine grid. - * \param[in] geo_coarse - Geometrical definition of the coarse grid. - * \param[in] config - Definition of the particular problem. - * \param[in] iMesh - Index of the mesh in multigrid computations. - */ - void RestrictResidualToCoarseGrid(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, - CGeometry *geo_coarse, CConfig *config, unsigned short iMesh); - /*! * \brief Add the truncation error to the residual. * \param[in] geometry - Geometrical definition of the problem. @@ -192,20 +180,6 @@ class CMultiGridIntegration final : public CIntegration { void GetProlongated_Correction(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config); - /*! - * \brief Prolongate the coarse-grid state correction back to the fine-grid residual correction. - * \param[in] RunTime_EqSystem - System of equations which is going to be solved. - * \param[in] sol_fine - Pointer to the solution on the fine grid. - * \param[in] sol_coarse - Pointer to the solution on the coarse grid. - * \param[in] geo_fine - Geometrical definition of the fine grid. - * \param[in] geo_coarse - Geometrical definition of the coarse grid. - * \param[in] config - Definition of the particular problem. - * \param[in] iMesh - Index of the mesh in multigrid computations. - */ - void ProlongateCorrectionToFineGrid(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, - CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config, - unsigned short iMesh); - /*! * \brief Do an implicit smoothing of the prolongated correction. * \param[in] RunTime_EqSystem - System of equations which is going to be solved. @@ -274,26 +248,6 @@ class CMultiGridIntegration final : public CIntegration { passivedouble lastRMS[2], char& exitReason, passivedouble& worstStepRatio, unsigned short& worstStep); - /*! - * \brief Restrict turbulent eddy viscosity from fine to coarser mesh levels. - * - * After a turbulence FAS V-cycle completes, this function volume-weights restricts - * mu_t from the finest mesh down to all coarser levels. The flow solver on the next - * outer iteration uses these restricted mu_t values at every coarse level for the - * eddy-viscosity coupling. This ensures consistency between flow and turbulence - * solutions across the multigrid hierarchy. - * - * \param[in] geometry - Geometry hierarchy for one zone/instance (all levels). - * \param[in] solver - Solver hierarchy for one zone/instance (all levels). - * \param[in] config - Problem configuration. - * \param[in] FinestMesh - Current finest active mesh index. - * \param[in] nMGLevels - Total number of MG levels. - */ - void RestrictTurbEddyViscToCoarseLevels(CGeometry** geometry, CSolver*** solver, - CConfig* config, - unsigned short FinestMesh, - unsigned short nMGLevels); - static constexpr int MAX_MG_LEVELS = 10; /*--- Early-exit smoothing state (shared across OMP threads via master write + barrier). ---*/ diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 12997293c87..0f9424191c6 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -184,46 +184,6 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, geometry[iZone][iInst][FinestMesh], config[iZone]); - /*--- Prolongate scalar solutions to the new finest mesh. - * All scalar solvers (turb, species, transition) run via SingleGrid_Iteration on - * GetFinestMesh(). Only turbulence additionally restricts its field downward to - * coarser meshes; no scalar ever propagates upward to finer meshes. Consequently - * meshes finer than FinestMesh hold their iter-0 startup values for the entire - * warmup phase. When FinestMesh is decremented these stale fields cause a large - * transient (e.g. +3 decade regression in rms[nu]). Prolongating here mirrors - * what SetProlongated_Solution does for the flow and eliminates the regression. ---*/ - if (config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { - SetProlongated_Solution(RUNTIME_TURB_SYS, - solver_container[iZone][iInst][FinestMesh-1][TURB_SOL], - solver_container[iZone][iInst][FinestMesh][TURB_SOL], - geometry[iZone][iInst][FinestMesh-1], - geometry[iZone][iInst][FinestMesh], - config[iZone]); - /*--- Recompute mu_t on the new finest mesh from the prolongated nu_tilde/k/omega. ---*/ - solver_container[iZone][iInst][FinestMesh-1][TURB_SOL]->Postprocessing( - geometry[iZone][iInst][FinestMesh-1], - solver_container[iZone][iInst][FinestMesh-1], - config[iZone], FinestMesh-1); - } - - if (config[iZone]->GetKind_Trans_Model() == TURB_TRANS_MODEL::LM) { - SetProlongated_Solution(RUNTIME_TRANS_SYS, - solver_container[iZone][iInst][FinestMesh-1][TRANS_SOL], - solver_container[iZone][iInst][FinestMesh][TRANS_SOL], - geometry[iZone][iInst][FinestMesh-1], - geometry[iZone][iInst][FinestMesh], - config[iZone]); - } - - if (config[iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { - SetProlongated_Solution(RUNTIME_SPECIES_SYS, - solver_container[iZone][iInst][FinestMesh-1][SPECIES_SOL], - solver_container[iZone][iInst][FinestMesh][SPECIES_SOL], - geometry[iZone][iInst][FinestMesh-1], - geometry[iZone][iInst][FinestMesh], - config[iZone]); - } - SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) } @@ -231,45 +191,11 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, FinestMesh = config[iZone]->GetFinestMesh(); - /*--- For turbulence MG: before descending to coarse levels, ensure mu_t is computed - * at the finest level and restricted to all coarser levels. This prevents inf - * residuals from coarse-level turbulence solves using stale/uninitialized mu_t. ---*/ - if (RunTime_EqSystem == RUNTIME_TURB_SYS && - config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { - - solver_container[iZone][iInst][FinestMesh][TURB_SOL]->Postprocessing( - geometry[iZone][iInst][FinestMesh], - solver_container[iZone][iInst][FinestMesh], - config[iZone], FinestMesh); - - RestrictTurbEddyViscToCoarseLevels(geometry[iZone][iInst], - solver_container[iZone][iInst], - config[iZone], FinestMesh, - config[iZone]->GetnMGLevels()); - } - /*--- Perform the Full Approximation Scheme multigrid ---*/ MultiGrid_Cycle(geometry, solver_container, numerics_container, config, FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); - /*--- After a turb FAS V-cycle: recompute mu_t at the finest active level from the updated - * nu_hat/k/omega and restrict it to all coarser levels. The flow FAS on the NEXT outer - * iteration uses these mu_t values at every coarse level for the eddy-viscosity coupling. - * (Postprocessing was already called on FinestMesh inside the last PreSmoothing step of - * MultiGrid_Cycle; we call it once more to be safe after the V-cycle correction is applied.) ---*/ - if (RunTime_EqSystem == RUNTIME_TURB_SYS && - config[iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE) { - solver_container[iZone][iInst][FinestMesh][TURB_SOL]->Postprocessing( - geometry[iZone][iInst][FinestMesh], - solver_container[iZone][iInst][FinestMesh], - config[iZone], FinestMesh); - RestrictTurbEddyViscToCoarseLevels(geometry[iZone][iInst], - solver_container[iZone][iInst], - config[iZone], FinestMesh, - config[iZone]->GetnMGLevels()); - } - /*--- Adapt coarse-grid CFL once per cycle using smoothing residuals gathered during the cycle. ---*/ const unsigned short nMGLevels = config[iZone]->GetnMGLevels(); BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS @@ -469,8 +395,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, PreSmoothing(RunTime_EqSystem, geometry, solver_container, config_container, solver_fine, numerics_fine, geometry_fine, solver_container_fine, config, iMesh, iZone, iRKLimit); - /*--- Assemble the coarse-grid FAS defect term by restricting the fine-grid residual defect, - * solving the coarse-grid state, and prolongating only the state correction back to the fine grid. ---*/ +/*--- Compute Forcing Term $P_(k+1) = I^(k+1)_k(P_k+F_k(u_k))-F_(k+1)(I^(k+1)_k u_k)$ and update solution for multigrid ---*/ if ( iMesh < config->GetnMGLevels() ) { @@ -508,23 +433,10 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, solver_coarse->Preprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem, false); - /*--- For turbulence: ensure flow primitives (density, laminar viscosity) are updated on the - * coarse level from the restricted conservative variables, THEN compute mu_t from the - * newly restricted turbulence variables. This ensures turbulence Postprocessing reads - * valid flow data and Space_Integration uses correct eddy viscosity. ---*/ - if (RunTime_EqSystem == RUNTIME_TURB_SYS && config->GetKind_Turb_Model() != TURB_MODEL::NONE) { - solver_container_coarse[FLOW_SOL]->Preprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1, NO_RK_ITER, RUNTIME_FLOW_SYS, false); - solver_coarse->Postprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1); - } Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); - /*--- Restrict the fine-grid residual defect to the coarse-grid FAS forcing term. ---*/ - if (RunTime_EqSystem == RUNTIME_FLOW_SYS) { - RestrictResidualToCoarseGrid(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); - } else { - SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); - } + SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); /*--- Restore the time integration settings. ---*/ @@ -546,12 +458,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, iMesh+1, nextRecurseParam, RunTime_EqSystem, iZone, iInst); } - /*--- Compute the coarse-grid state correction and prolongate it back to the fine grid. ---*/ - if (RunTime_EqSystem == RUNTIME_FLOW_SYS) { - ProlongateCorrectionToFineGrid(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh); - } else { - GetProlongated_Correction(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); - } + GetProlongated_Correction(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); const auto& mgOpts = config->GetMGOptions(); SmoothProlongated_Correction(RunTime_EqSystem, solver_fine, geometry_fine, mgOpts.MG_CorrecSmooth[iMesh], mgOpts.MG_Smooth_Coeff, config, iMesh); @@ -912,11 +819,6 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ if (config->GetMGOptions().MG_Smooth_Output) { const su2double res = sqrt(solver->LinSysRes.squaredNorm() / (nVar * geometry->GetGlobal_nPointDomain())); SU2_OMP_SAFE_GLOBAL_ACCESS(lastCorrecSmoothRMS[iMesh][1] = SU2_TYPE::GetValue(res);) - - if (SU2_MPI::GetRank() == MASTER_NODE && use_conservative_damping) { - cout << "[MG CORR-SMOOTH] turbulence nSmooth=" << val_nSmooth - << " norm=" << res << "\n"; - } } } @@ -1131,32 +1033,6 @@ void CMultiGridIntegration::SetResidual_Term(CGeometry *geometry, CSolver *solve } -void CMultiGridIntegration::RestrictResidualToCoarseGrid(CSolver *sol_fine, CSolver *sol_coarse, - CGeometry *geo_fine, CGeometry *geo_coarse, - CConfig *config, unsigned short iMesh) { - SU2_ZONE_SCOPED - - /*--- This is the standard FAS restriction step: the fine-grid defect is passed to the - * coarse-grid problem as a forcing term. The existing SetForcing_Term routine already - * implements the conservative volume-weighted transfer and the damping factor in the - * same way the original MG cycle expects. ---*/ - SetForcing_Term(sol_fine, sol_coarse, geo_fine, geo_coarse, config, iMesh); -} - -void CMultiGridIntegration::ProlongateCorrectionToFineGrid(unsigned short RunTime_EqSystem, CSolver *sol_fine, - CSolver *sol_coarse, CGeometry *geo_fine, - CGeometry *geo_coarse, CConfig *config, - unsigned short iMesh) { - SU2_ZONE_SCOPED - - /*--- This is the standard FAS prolongation step: build the coarse-grid state correction, - * then transfer that correction to the fine-grid residual correction. The original - * GetProlongated_Correction routine already performs this transfer in the correct form; - * the additional scaling here would be equivalent to changing the correction operator. - * Keep the transfer operator unchanged and let the existing damping path control the size. ---*/ - GetProlongated_Correction(RunTime_EqSystem, sol_fine, sol_coarse, geo_fine, geo_coarse, config); -} - void CMultiGridIntegration::SetRestricted_Solution(unsigned short RunTime_EqSystem, CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config) { SU2_ZONE_SCOPED @@ -1350,36 +1226,3 @@ void CMultiGridIntegration::Adjoint_Setup(CGeometry ****geometry, CSolver *****s } } - -void CMultiGridIntegration::RestrictTurbEddyViscToCoarseLevels(CGeometry** geometry, CSolver*** solver, - CConfig* config, - unsigned short FinestMesh, - unsigned short nMGLevels) { - SU2_ZONE_SCOPED - - for (unsigned short iMesh = FinestMesh; iMesh < nMGLevels; iMesh++) { - - CGeometry* geo_fine = geometry[iMesh]; - CGeometry* geo_coarse = geometry[iMesh + 1]; - CSolver* sol_fine = solver[iMesh][TURB_SOL]; - CSolver* sol_coarse = solver[iMesh + 1][TURB_SOL]; - - /*--- Volume-weighted restriction of mu_t from fine to coarse. ---*/ - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) - for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { - - const su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); - su2double EddyVisc = 0.0; - - for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { - auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); - su2double Area_Children = geo_fine->nodes->GetVolume(Point_Fine); - su2double mu_t = sol_fine->GetNodes()->GetmuT(Point_Fine); - EddyVisc += mu_t * Area_Children / Area_Parent; - } - - sol_coarse->GetNodes()->SetmuT(Point_Coarse, EddyVisc); - } - END_SU2_OMP_FOR - } -} diff --git a/SU2_CFD/src/iteration/CFluidIteration.cpp b/SU2_CFD/src/iteration/CFluidIteration.cpp index bdfd71fb8fc..c3e95355858 100644 --- a/SU2_CFD/src/iteration/CFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CFluidIteration.cpp @@ -82,7 +82,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe integration[val_iZone][val_iInst][FLOW_SOL]->MultiGrid_Iteration(geometry, solver, numerics, config, RUNTIME_FLOW_SYS, val_iZone, val_iInst); - /*--- If the flow integration is not fully coupled, run the various single/multi-grid integrations. ---*/ + /*--- If the flow integration is not fully coupled, run the various single-grid integrations. ---*/ if (config[val_iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE && !frozen_visc) { @@ -95,14 +95,8 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe } /*--- Solve the turbulence model ---*/ - /*--- Use multigrid if MG_TURB is enabled, otherwise use single-grid. ---*/ config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_TURB_SYS); - - if (config[val_iZone]->GetMGOptions().TurbMG) { - integration[val_iZone][val_iInst][TURB_SOL]->MultiGrid_Iteration(geometry, solver, numerics, config, - RUNTIME_TURB_SYS, val_iZone, val_iInst); - } else { integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_TURB_SYS, val_iZone, val_iInst); } @@ -110,7 +104,6 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe if (config[val_iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_SPECIES_SYS); - /*--- Use multigrid if MG_SPECIES is enabled (future feature), otherwise use single-grid. ---*/ integration[val_iZone][val_iInst][SPECIES_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_SPECIES_SYS, val_iZone, val_iInst); @@ -127,7 +120,6 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe if (config[val_iZone]->GetWeakly_Coupled_Heat()) { config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_HEAT_SYS); - /*--- Use multigrid if MG_HEAT is enabled (future feature), otherwise use single-grid. ---*/ integration[val_iZone][val_iInst][HEAT_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_HEAT_SYS, val_iZone, val_iInst); } diff --git a/SU2_CFD/src/solvers/CSolverFactory.cpp b/SU2_CFD/src/solvers/CSolverFactory.cpp index 00e7064d31c..ee798c1384b 100644 --- a/SU2_CFD/src/solvers/CSolverFactory.cpp +++ b/SU2_CFD/src/solvers/CSolverFactory.cpp @@ -313,7 +313,7 @@ CSolver* CSolverFactory::CreateSubSolver(SUB_SOLVER_TYPE kindSolver, CSolver **s case SUB_SOLVER_TYPE::TURB_SA: case SUB_SOLVER_TYPE::TURB_SST: genericSolver = CreateTurbSolver(kindTurbModel, solver, geometry, config, iMGLevel, false); - metaData.integrationType = config->GetMGOptions().TurbMG ? INTEGRATION_TYPE::MULTIGRID : INTEGRATION_TYPE::SINGLEGRID; + metaData.integrationType = INTEGRATION_TYPE::SINGLEGRID; break; case SUB_SOLVER_TYPE::TEMPLATE: genericSolver = new CTemplateSolver(geometry, config); diff --git a/SU2_CFD/src/solvers/CTurbSASolver.cpp b/SU2_CFD/src/solvers/CTurbSASolver.cpp index 714d1972695..b58afe4c9b3 100644 --- a/SU2_CFD/src/solvers/CTurbSASolver.cpp +++ b/SU2_CFD/src/solvers/CTurbSASolver.cpp @@ -65,7 +65,7 @@ CTurbSASolver::CTurbSASolver(CGeometry *geometry, CConfig *config, unsigned shor /*--- Single grid simulation ---*/ - if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL || config->GetMGOptions().TurbMG) { + if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL) { /*--- Define some auxiliar vector related with the residual ---*/ diff --git a/SU2_CFD/src/solvers/CTurbSSTSolver.cpp b/SU2_CFD/src/solvers/CTurbSSTSolver.cpp index 97b4ee1d78c..579b5f3cf72 100644 --- a/SU2_CFD/src/solvers/CTurbSSTSolver.cpp +++ b/SU2_CFD/src/solvers/CTurbSSTSolver.cpp @@ -59,7 +59,7 @@ CTurbSSTSolver::CTurbSSTSolver(CGeometry *geometry, CConfig *config, unsigned sh /*--- Single grid simulation ---*/ - if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL || config->GetMGOptions().TurbMG) { + if (iMesh == MESH_0 || config->GetMGCycle() == MG_CYCLE::FULL) { /*--- Define some auxiliary vector related with the residual ---*/ diff --git a/SU2_CFD/src/variables/CTurbVariable.cpp b/SU2_CFD/src/variables/CTurbVariable.cpp index 62b87f9363d..139223055a4 100644 --- a/SU2_CFD/src/variables/CTurbVariable.cpp +++ b/SU2_CFD/src/variables/CTurbVariable.cpp @@ -35,18 +35,6 @@ CTurbVariable::CTurbVariable(unsigned long npoint, unsigned long ndim, unsigned turb_index.resize(nPoint) = su2double(1.0); intermittency.resize(nPoint) = su2double(1.0); - /*--- Allocate residual structures for multigrid (required for turbulence MG). ---*/ - Res_TruncError.resize(nPoint, nVar) = su2double(0.0); - - /*--- Allocate smoothing arrays if correction smoothing is enabled at any MG level. ---*/ - for (unsigned long iMesh = 0; iMesh <= config->GetnMGLevels(); iMesh++) { - if (config->GetMGOptions().MG_CorrecSmooth[iMesh] > 0) { - Residual_Sum.resize(nPoint, nVar); - Residual_Old.resize(nPoint, nVar); - break; - } - } - } void CTurbVariable::RegisterEddyViscosity(bool input) { diff --git a/config_template.cfg b/config_template.cfg index 2c7d882c11f..45a822755c9 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1732,10 +1732,6 @@ MG_IMPLICIT_LINES_ISOTROPIC= NO % Number of iterations on coarsest mesh during Full Multigrid (FMG) startup phase. % After this many iterations, solution is prolongated to finer mesh (default 100). MG_STARTUP_ITER= 100 -% -% Run turbulence equations through a FAS Multigrid V-cycle (YES, NO) -% When disabled, turbulence is solved with single-grid only (default NO). -MG_TURB= NO % -------------------------- MESH SMOOTHING -----------------------------% % From 4922e7b675398d8b1d93804dda3c1b554d5ebc26 Mon Sep 17 00:00:00 2001 From: Nijso Date: Tue, 4 Aug 2026 21:44:49 +0200 Subject: [PATCH 07/54] Apply suggestions from code review Co-authored-by: Nijso --- .../src/integration/CMultiGridIntegration.cpp | 80 ++----------------- SU2_CFD/src/iteration/CFluidIteration.cpp | 4 +- 2 files changed, 7 insertions(+), 77 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 0f9424191c6..74d2b2e75df 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -395,7 +395,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, PreSmoothing(RunTime_EqSystem, geometry, solver_container, config_container, solver_fine, numerics_fine, geometry_fine, solver_container_fine, config, iMesh, iZone, iRKLimit); -/*--- Compute Forcing Term $P_(k+1) = I^(k+1)_k(P_k+F_k(u_k))-F_(k+1)(I^(k+1)_k u_k)$ and update solution for multigrid ---*/ + /*--- Compute Forcing Term $P_(k+1) = I^(k+1)_k(P_k+F_k(u_k))-F_(k+1)(I^(k+1)_k u_k)$ and update solution for multigrid ---*/ if ( iMesh < config->GetnMGLevels() ) { @@ -418,16 +418,15 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, Space_Integration(geometry_fine, solver_container_fine, numerics_fine, config, iMesh, NO_RK_ITER, RunTime_EqSystem); - /*--- LinSysRes = R(u_N) here, before the fine-grid defect term is assembled. ---*/ + /*--- LinSysRes = R(u_N) here, before tau is added by SetResidual_Term. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { lastPreSmoothRMS[iMesh][1] = ComputeLinSysResRMS(solver_fine); } END_SU2_OMP_SAFE_GLOBAL_ACCESS - /*--- Assemble the fine-grid defect term that will be restricted to the coarse-grid FAS problem. ---*/ SetResidual_Term(geometry_fine, solver_fine); - /*--- Restrict the fine-grid state to the coarse grid and initialize the coarse-grid state. ---*/ + /*--- Compute $r_(k+1) = F_(k+1)(I^(k+1)_k u_k)$ ---*/ SetRestricted_Solution(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); @@ -435,6 +434,8 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); + + /*--- Compute $P_(k+1) = I^(k+1)_k(r_k) - r_(k+1) ---*/ SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); @@ -883,77 +884,6 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet } END_SU2_OMP_FOR - /*--- DIAGNOSTIC: log the max applied correction (factor * LinSysRes) at fine-grid wall points - * vs interior. ---*/ - if (config->GetMGOptions().MG_Smooth_Output && SU2_MPI::GetRank() == MASTER_NODE) { - BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS - { - if (nVar > 2) { - su2double maxWall0 = 0.0, maxWallN = 0.0, maxWallMom = 0.0; - su2double maxInter0 = 0.0, maxInterN = 0.0, maxInterMom = 0.0; - su2double maxWallApply0 = 0.0, maxWallApplyN = 0.0, maxWallApplyMom = 0.0; - su2double maxInterApply0 = 0.0, maxInterApplyN = 0.0, maxInterApplyMom = 0.0; - - for (auto iPoint = 0ul; iPoint < geo_fine->GetnPointDomain(); iPoint++) { - const auto* corr = sol_fine->LinSysRes.GetBlock(iPoint); - const su2double localDamping = isWall[iPoint] ? wall_damping : base_damping; - const su2double applied0 = fabs(localDamping * factor * corr[0]); - const su2double appliedN = fabs(localDamping * factor * corr[nVar-1]); - su2double appliedMom = 0.0; - for (auto iVar = 1u; iVar < static_cast(nVar-1); iVar++) { - appliedMom = max(appliedMom, fabs(localDamping * factor * corr[iVar])); - } - - if (isWall[iPoint]) { - maxWall0 = max(maxWall0, fabs(factor * corr[0])); - maxWallN = max(maxWallN, fabs(factor * corr[nVar-1])); - maxWallMom = max(maxWallMom, fabs(factor * corr[0])); - maxWallApply0 = max(maxWallApply0, applied0); - maxWallApplyN = max(maxWallApplyN, appliedN); - maxWallApplyMom = max(maxWallApplyMom, appliedMom); - } else { - maxInter0 = max(maxInter0, fabs(factor * corr[0])); - maxInterN = max(maxInterN, fabs(factor * corr[nVar-1])); - maxInterMom = max(maxInterMom, fabs(factor * corr[0])); - maxInterApply0 = max(maxInterApply0, applied0); - maxInterApplyN = max(maxInterApplyN, appliedN); - maxInterApplyMom = max(maxInterApplyMom, appliedMom); - } - } - auto ratio = [](su2double w, su2double i) { return (i > 1e-30) ? w/i : 0.0; }; - cout << "[MG APPL L" << iMesh << " wall/inter] rho=" << ratio(maxWallApply0, maxInterApply0) - << " mom=" << ratio(maxWallApplyMom, maxInterApplyMom) - << " E=" << ratio(maxWallApplyN, maxInterApplyN) - << " (raw wall/inter: rho=" << maxWall0 << "/" << maxInter0 - << ", E=" << maxWallN << "/" << maxInterN - << "; applied wall/inter: rho=" << maxWallApply0 << "/" << maxInterApply0 - << ", E=" << maxWallApplyN << "/" << maxInterApplyN - << "; levelScale=" << levelScale << ", damp(wall/inter)=" << wall_damping << "/" << base_damping << ")\n"; - } else { - su2double maxWall = 0.0, maxInter = 0.0; - su2double maxWallApply = 0.0, maxInterApply = 0.0; - for (auto iPoint = 0ul; iPoint < geo_fine->GetnPointDomain(); iPoint++) { - const auto* corr = sol_fine->LinSysRes.GetBlock(iPoint); - const su2double localDamping = isWall[iPoint] ? wall_damping : base_damping; - const su2double mag = fabs(factor * corr[0]); - const su2double appliedMag = fabs(localDamping * factor * corr[0]); - if (isWall[iPoint]) { - maxWall = max(maxWall, mag); - maxWallApply = max(maxWallApply, appliedMag); - } else { - maxInter = max(maxInter, mag); - maxInterApply = max(maxInterApply, appliedMag); - } - } - cout << "[MG TURB APPLY L" << iMesh << "] damp(wall/interior)= " << wall_damping << "/" << base_damping - << " raw max(wall/interior)= " << maxWall << "/" << maxInter - << " applied max(wall/interior)= " << maxWallApply << "/" << maxInterApply - << " levelScale=" << levelScale << "\n"; - } - } - END_SU2_OMP_SAFE_GLOBAL_ACCESS - } - /*--- MPI the new interpolated solution ---*/ sol_fine->InitiateComms(geo_fine, config, MPI_QUANTITIES::SOLUTION); diff --git a/SU2_CFD/src/iteration/CFluidIteration.cpp b/SU2_CFD/src/iteration/CFluidIteration.cpp index c3e95355858..89fee917a5c 100644 --- a/SU2_CFD/src/iteration/CFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CFluidIteration.cpp @@ -82,7 +82,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe integration[val_iZone][val_iInst][FLOW_SOL]->MultiGrid_Iteration(geometry, solver, numerics, config, RUNTIME_FLOW_SYS, val_iZone, val_iInst); - /*--- If the flow integration is not fully coupled, run the various single-grid integrations. ---*/ + /*--- If the flow integration is not fully coupled, run the various single grid integrations. ---*/ if (config[val_iZone]->GetKind_Turb_Model() != TURB_MODEL::NONE && !frozen_visc) { @@ -97,7 +97,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe /*--- Solve the turbulence model ---*/ config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_TURB_SYS); - integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, + integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_TURB_SYS, val_iZone, val_iInst); } } From 5d806798b303aa07d99fa2dabe45e62f72ded981 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Tue, 4 Aug 2026 21:47:10 +0200 Subject: [PATCH 08/54] precommit --- Common/src/geometry/CMultiGridGeometry.cpp | 102 ++++++++++++--------- SU2_CFD/src/iteration/CFluidIteration.cpp | 1 - 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index c9192e17afd..feeb1e2c60a 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -437,11 +437,16 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long n_corrupted_implicit_CVs = 0; for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) nCVs_1child++; - else if (nChildren == 2) nCVs_2child++; - else if (nChildren == 3) nCVs_3child++; - else if (nChildren == 4) nCVs_4child++; - else nCVs_other++; + if (nChildren == 1) + nCVs_1child++; + else if (nChildren == 2) + nCVs_2child++; + else if (nChildren == 3) + nCVs_3child++; + else if (nChildren == 4) + nCVs_4child++; + else + nCVs_other++; if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { n_corrupted_implicit_CVs++; @@ -459,9 +464,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (n_corrupted_implicit_CVs > 0) { cout << " AFTER DOMAIN AGGLOMERATION: " << n_corrupted_implicit_CVs << " implicit line CVs were corrupted (child count != 2)" << endl; - cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child - << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child - << ", 4-child=" << nCVs_4child; + cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child + << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; if (nCVs_other > 0) cout << ", other=" << nCVs_other; cout << endl; } @@ -546,11 +550,16 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long n_corrupted_after_hanging = 0; for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) nCVs_1child++; - else if (nChildren == 2) nCVs_2child++; - else if (nChildren == 3) nCVs_3child++; - else if (nChildren == 4) nCVs_4child++; - else nCVs_other++; + if (nChildren == 1) + nCVs_1child++; + else if (nChildren == 2) + nCVs_2child++; + else if (nChildren == 3) + nCVs_3child++; + else if (nChildren == 4) + nCVs_4child++; + else + nCVs_other++; if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { n_corrupted_after_hanging++; @@ -559,9 +568,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (n_corrupted_after_hanging > 0) { cout << " AFTER HANGING NODE CORRECTION: " << n_corrupted_after_hanging << " implicit line CVs corrupted (child count != 2)" << endl; - cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child - << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child - << ", 4-child=" << nCVs_4child; + cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child + << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; if (nCVs_other > 0) cout << ", other=" << nCVs_other; cout << endl; } @@ -575,20 +583,25 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; for (auto iCV = 0ul; iCV < nPointDomain; iCV++) { const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) nCVs_1child++; - else if (nChildren == 2) nCVs_2child++; - else if (nChildren == 3) nCVs_3child++; - else if (nChildren == 4) nCVs_4child++; - else nCVs_other++; + if (nChildren == 1) + nCVs_1child++; + else if (nChildren == 2) + nCVs_2child++; + else if (nChildren == 3) + nCVs_3child++; + else if (nChildren == 4) + nCVs_4child++; + else + nCVs_other++; } - cout << " CV distribution: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child - << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; + cout << " CV distribution: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child + << ", 4-child=" << nCVs_4child; if (nCVs_other > 0) cout << ", other=" << nCVs_other; cout << endl; if (nCVs_3child > 0 || (!config->GetMGOptions().MG_Implicit_Lines_Isotropic && nCVs_4child > 0)) { - cout << " WARNING: Detected unexpected CV child counts (3-child=" << nCVs_3child - << ", 4-child=" << nCVs_4child << " in ANISO mode)" << endl; + cout << " WARNING: Detected unexpected CV child counts (3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child + << " in ANISO mode)" << endl; } } @@ -1378,9 +1391,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; const unsigned long nPointFine = fine_grid->GetnPoint(); - const unsigned long starting_Index_CoarseCV = Index_CoarseCV; /*--- Track how many CVs we create ---*/ - const bool DEBUG_OUTPUT = (rank == MASTER_NODE); /*--- Enable detailed diagnostic output ---*/ - const unsigned long DEBUG_CV_LIMIT = 20; /*--- Show details for first N CVs ---*/ + const unsigned long starting_Index_CoarseCV = Index_CoarseCV; /*--- Track how many CVs we create ---*/ + const bool DEBUG_OUTPUT = (rank == MASTER_NODE); /*--- Enable detailed diagnostic output ---*/ + const unsigned long DEBUG_CV_LIMIT = 20; /*--- Show details for first N CVs ---*/ /*--- Collect implicit lines starting at viscous (no-slip) wall vertices only. * Seeding from non-wall boundaries (farfield, inlet, outlet, symmetry) would @@ -1484,7 +1497,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, avg_len += L.size(); } if (!lines.empty()) avg_len /= lines.size(); - cout << " Line lengths: min=" << min_len << ", max=" << max_len << ", avg=" << std::setprecision(1) << std::fixed << avg_len << endl; + cout << " Line lengths: min=" << min_len << ", max=" << max_len << ", avg=" << std::setprecision(1) << std::fixed + << avg_len << endl; /*--- Show first few lines for debugging ---*/ cout << " First 5 lines (showing first 4 nodes):" << endl; @@ -1597,8 +1611,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { const auto* coord_a = fine_grid->nodes->GetCoord(a); const auto* coord_b = fine_grid->nodes->GetCoord(b); - cout << " CV " << Index_CoarseCV << " (ISO): nodes " << a << "+" << b << "+" << c << "+" << d - << " | lines[" << li1 << "][" << idx1 << "," << idx2 << "]+lines[" << li2_best << "][" << idx1 << "," << idx2 << "]" + cout << " CV " << Index_CoarseCV << " (ISO): nodes " << a << "+" << b << "+" << c << "+" << d << " | lines[" + << li1 << "][" << idx1 << "," << idx2 << "]+lines[" << li2_best << "][" << idx1 << "," << idx2 << "]" << " | coord_a=(" << coord_a[0] << "," << coord_a[1] << ")" << " coord_b=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; } @@ -1670,17 +1684,20 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto wall_b = lines[li2_best][0]; const auto* coord_wall_a = fine_grid->nodes->GetCoord(wall_a); const auto* coord_wall_b = fine_grid->nodes->GetCoord(wall_b); - su2double wall_dist = sqrt(pow(coord_wall_a[0] - coord_wall_b[0], 2) + - pow(coord_wall_a[1] - coord_wall_b[1], 2)); + su2double wall_dist = + sqrt(pow(coord_wall_a[0] - coord_wall_b[0], 2) + pow(coord_wall_a[1] - coord_wall_b[1], 2)); /*--- Check if wall vertices are neighbors ---*/ bool walls_are_neighbors = false; for (auto neighbor : fine_grid->nodes->GetPoints(wall_a)) { - if (neighbor == wall_b) { walls_are_neighbors = true; break; } + if (neighbor == wall_b) { + walls_are_neighbors = true; + break; + } } - cout << " Pairing lines " << li1 << " + " << li2_best << " at pos=" << pos - << " | wall_dist=" << wall_dist << " | walls_neighbors=" << (walls_are_neighbors ? "YES" : "NO") << endl; + cout << " Pairing lines " << li1 << " + " << li2_best << " at pos=" << pos << " | wall_dist=" << wall_dist + << " | walls_neighbors=" << (walls_are_neighbors ? "YES" : "NO") << endl; } /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ @@ -1697,12 +1714,15 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, su2double dist = sqrt(pow(coord_a[0] - coord_b[0], 2) + pow(coord_a[1] - coord_b[1], 2)); bool are_neighbors = false; for (auto neighbor : fine_grid->nodes->GetPoints(a)) { - if (neighbor == b) { are_neighbors = true; break; } + if (neighbor == b) { + are_neighbors = true; + break; + } } - cout << " CV " << Index_CoarseCV << " (ANISO): nodes " << a << "+" << b - << " | lines[" << li1 << "][" << pos << "]+lines[" << li2_best << "][" << pos << "]" - << " | dist=" << dist << " | neighbors=" << (are_neighbors ? "YES" : "NO") - << " | coords A=(" << coord_a[0] << "," << coord_a[1] << ")" + cout << " CV " << Index_CoarseCV << " (ANISO): nodes " << a << "+" << b << " | lines[" << li1 << "][" << pos + << "]+lines[" << li2_best << "][" << pos << "]" + << " | dist=" << dist << " | neighbors=" << (are_neighbors ? "YES" : "NO") << " | coords A=(" + << coord_a[0] << "," << coord_a[1] << ")" << " B=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; } diff --git a/SU2_CFD/src/iteration/CFluidIteration.cpp b/SU2_CFD/src/iteration/CFluidIteration.cpp index 89fee917a5c..264effbdf7a 100644 --- a/SU2_CFD/src/iteration/CFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CFluidIteration.cpp @@ -99,7 +99,6 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_TURB_SYS); integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, RUNTIME_TURB_SYS, val_iZone, val_iInst); - } } if (config[val_iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { From 7a0db1007efba030850b98a9978c9fcf2158b922 Mon Sep 17 00:00:00 2001 From: Nijso Date: Tue, 4 Aug 2026 21:50:27 +0200 Subject: [PATCH 09/54] Apply suggestions from code review Co-authored-by: Nijso --- SU2_CFD/src/integration/CMultiGridIntegration.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 74d2b2e75df..fae22207943 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -748,8 +748,6 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ if (val_nSmooth == 0) return; const unsigned short nVar = solver->GetnVar(); - const bool use_conservative_damping = (nVar <= 2); - const su2double turbulence_base_damping = 0.50; SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); iPoint++) { @@ -789,11 +787,8 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ const auto* Residual_Sum = solver->GetNodes()->GetResidual_Sum(iPoint); const auto* Residual_Old = solver->GetNodes()->GetResidual_Old(iPoint); - for (auto iVar = 0u; iVar < nVar; iVar++) { - su2double smoothed = (Residual_Old[iVar] + val_smooth_coeff*Residual_Sum[iVar])*factor; - if (use_conservative_damping) smoothed *= turbulence_base_damping; - solver->LinSysRes(iPoint,iVar) = smoothed; - } + for (auto iVar = 0u; iVar < nVar; iVar++) + solver->LinSysRes(iPoint,iVar) = (Residual_Old[iVar] + val_smooth_coeff*Residual_Sum[iVar])*factor; } END_SU2_OMP_FOR @@ -828,10 +823,6 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet SU2_ZONE_SCOPED const unsigned short nVar = sol_fine->GetnVar(); - const bool use_conservative_damping = (nVar <= 2); - const su2double levelScale = GetMGLevelCorrectionScale(iMesh); - const su2double base_damping = use_conservative_damping ? max(su2double{0.15}, 0.50 * levelScale) : 1.0; - const su2double wall_damping = use_conservative_damping ? max(su2double{0.10}, 0.25 * levelScale) : 1.0; /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); From aca34afd1a21a986112adc7a1952b67f6f570643 Mon Sep 17 00:00:00 2001 From: Nijso Date: Tue, 4 Aug 2026 21:55:35 +0200 Subject: [PATCH 10/54] Apply suggestions from code review Co-authored-by: Nijso --- .../src/integration/CMultiGridIntegration.cpp | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index fae22207943..a14ad116f03 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -827,49 +827,16 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); - vector isWall(geo_fine->GetnPoint(), false); - for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) - if (config->GetViscous_Wall(iMarker)) - for (auto iVertex = 0ul; iVertex < geo_fine->nVertex[iMarker]; iVertex++) - isWall[geo_fine->vertex[iMarker][iVertex]->GetNode()] = true; - SU2_OMP_FOR_STAT(roundUpDiv(geo_fine->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Fine = 0ul; Point_Fine < geo_fine->GetnPointDomain(); Point_Fine++) { auto* Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); auto* Solution_Fine = sol_fine->GetNodes()->GetSolution(Point_Fine); - - su2double residualMag = 0.0; - su2double correctionMag = 0.0; for (auto iVar = 0u; iVar < nVar; iVar++) { /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ - if (Residual_Fine[iVar] != Residual_Fine[iVar]) { - Residual_Fine[iVar] = 0.0; - } - - const su2double corr = factor * Residual_Fine[iVar]; - residualMag = max(residualMag, fabs(Residual_Fine[iVar])); - correctionMag = max(correctionMag, fabs(corr)); - } - - su2double correctionScale = 1.0; - constexpr su2double maxAllowedRatio = 1.25; - if (residualMag > 1e-30 && correctionMag > 1e-30) { - const su2double ratio = correctionMag / residualMag; - if (ratio > maxAllowedRatio) { - correctionScale = maxAllowedRatio / ratio; - } - } + if (Residual_Fine[iVar] != Residual_Fine[iVar]) +Residual_Fine[iVar] = 0.0; - const su2double localDamping = use_conservative_damping ? (isWall[Point_Fine] ? wall_damping : base_damping) : 1.0; - for (auto iVar = 0u; iVar < nVar; iVar++) { su2double correction = factor * Residual_Fine[iVar]; - correction *= localDamping; - correction *= correctionScale; - - if (!std::isfinite(correction)) { - correction = 0.0; - } - Solution_Fine[iVar] += correction; } } From 5648a3c54af211e7c435feb48fe6370a04daa9b0 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 09:26:10 +0200 Subject: [PATCH 11/54] fix cfl for full multigrid --- .../src/integration/CMultiGridIntegration.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index a14ad116f03..b873605108e 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -185,6 +185,23 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, config[iZone]); SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) + + auto nMG = config[iZone]->GetnMGLevels(); + auto cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)); + auto cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; + cout << "level =" << nMG - FinestMesh << endl; + cout <<"finestmesh = " << FinestMesh << endl; + cout << "CFL = " << cfl_base << ", scaling factor = " << cflScaling[FinestMesh-1] << endl; + // now scale the cfl using the scaling factors by multiplying from the first factor up to the factor for the current level + for (unsigned short iMesh = 0; iMesh < FinestMesh-1; ++iMesh) { + cout << "scale = " << cflScaling[iMesh] << endl; + cfl_base *= cflScaling[iMesh]; + } + cout << "new CFL = " << cfl_base << endl; + // set the new base cfl for MESH_0 to the scaled value: + SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SetCFL(MESH_0, cfl_base);) + + } /*--- Set the current finest grid (full multigrid strategy) ---*/ @@ -434,7 +451,7 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); - + /*--- Compute $P_(k+1) = I^(k+1)_k(r_k) - r_(k+1) ---*/ SetForcing_Term(solver_fine, solver_coarse, geometry_fine, geometry_coarse, config, iMesh+1); From 08a2f028eb9c6c75eb1a7eff29eec0b5c528668e Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 11:08:23 +0200 Subject: [PATCH 12/54] fix cfl for full multigrid --- SU2_CFD/include/solvers/CSolver.hpp | 6 +++ .../src/integration/CMultiGridIntegration.cpp | 54 ++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/SU2_CFD/include/solvers/CSolver.hpp b/SU2_CFD/include/solvers/CSolver.hpp index 942e2e25877..9d86e6afc5f 100644 --- a/SU2_CFD/include/solvers/CSolver.hpp +++ b/SU2_CFD/include/solvers/CSolver.hpp @@ -389,6 +389,12 @@ class CSolver { */ inline su2double GetAvg_CFL_Local(void) const { return Avg_CFL_Local; } + /*! + * \brief Set the value of the average local CFL number. + * \param[in] val_cfl - Average CFL number. + */ + inline void SetAvg_CFL_Local(su2double val_cfl) { Avg_CFL_Local = val_cfl; } + /*! * \brief Get the number of variables of the problem. */ diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index b873605108e..6a5471024f1 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -31,6 +31,7 @@ #include #include #include +#include using namespace std; @@ -67,6 +68,24 @@ static su2double GetMGLevelCorrectionScale(unsigned short iMesh) { } } +static passivedouble GetWarmupCFLScale(const std::vector& cflScaling, + unsigned long innerIter, + unsigned long startupIter) { + if (startupIter == 0 || cflScaling.empty()) return 1.0; + + const passivedouble firstScale = max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling.front())); + if (innerIter < startupIter) { + if (cflScaling.size() > 1) { + const passivedouble secondScale = max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[1])); + return firstScale * secondScale; + } + return firstScale; + } + + if (innerIter < 2 * startupIter) return firstScale; + return 1.0; +} + inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { passivedouble result = 0; for (unsigned short iVar = 0; iVar < solver->GetnVar(); ++iVar) { @@ -186,20 +205,29 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) - auto nMG = config[iZone]->GetnMGLevels(); - auto cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)); - auto cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; - cout << "level =" << nMG - FinestMesh << endl; - cout <<"finestmesh = " << FinestMesh << endl; - cout << "CFL = " << cfl_base << ", scaling factor = " << cflScaling[FinestMesh-1] << endl; - // now scale the cfl using the scaling factors by multiplying from the first factor up to the factor for the current level - for (unsigned short iMesh = 0; iMesh < FinestMesh-1; ++iMesh) { - cout << "scale = " << cflScaling[iMesh] << endl; - cfl_base *= cflScaling[iMesh]; + /*--- Full-MG warmup: seed the history CFL with the base CFL scaled by the + * current warmup phase, so Avg_CFL_Local matches what the solver uses. ---*/ + static std::map base_cfl_by_zone; + if (base_cfl_by_zone.find(iZone) == base_cfl_by_zone.end()) { + base_cfl_by_zone[iZone] = SU2_TYPE::GetValue(config[iZone]->GetCFL(MESH_0)); } - cout << "new CFL = " << cfl_base << endl; - // set the new base cfl for MESH_0 to the scaled value: - SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SetCFL(MESH_0, cfl_base);) + + const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; + const unsigned long innerIter = config[iZone]->GetInnerIter(); + const passivedouble cfl_base = base_cfl_by_zone[iZone]; + const passivedouble warmupScale = GetWarmupCFLScale(cflScaling, innerIter, startup_iter); + const passivedouble warmupCFL = cfl_base * warmupScale; + + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS + { + config[iZone]->SetCFL(MESH_0, warmupCFL); + CGeometry* geo_c = geometry[iZone][iInst][MESH_0]; + CSolver* sol_c = solver_container[iZone][iInst][MESH_0][Solver_Position]; + sol_c->SetAvg_CFL_Local(warmupCFL); + for (auto iPoint = 0ul; iPoint < geo_c->GetnPoint(); iPoint++) + sol_c->GetNodes()->SetLocalCFL(iPoint, warmupCFL); + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS } From aac68022e0f83abd2759e03d96304e22394d7e30 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 11:33:29 +0200 Subject: [PATCH 13/54] fix cfl for full multigrid --- .../src/integration/CMultiGridIntegration.cpp | 70 ++++++------------- 1 file changed, 22 insertions(+), 48 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 6a5471024f1..1697356a817 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -204,64 +204,33 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, config[iZone]); SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) - - /*--- Full-MG warmup: seed the history CFL with the base CFL scaled by the - * current warmup phase, so Avg_CFL_Local matches what the solver uses. ---*/ - static std::map base_cfl_by_zone; - if (base_cfl_by_zone.find(iZone) == base_cfl_by_zone.end()) { - base_cfl_by_zone[iZone] = SU2_TYPE::GetValue(config[iZone]->GetCFL(MESH_0)); - } - - const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; - const unsigned long innerIter = config[iZone]->GetInnerIter(); - const passivedouble cfl_base = base_cfl_by_zone[iZone]; - const passivedouble warmupScale = GetWarmupCFLScale(cflScaling, innerIter, startup_iter); - const passivedouble warmupCFL = cfl_base * warmupScale; - - BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS - { - config[iZone]->SetCFL(MESH_0, warmupCFL); - CGeometry* geo_c = geometry[iZone][iInst][MESH_0]; - CSolver* sol_c = solver_container[iZone][iInst][MESH_0][Solver_Position]; - sol_c->SetAvg_CFL_Local(warmupCFL); - for (auto iPoint = 0ul; iPoint < geo_c->GetnPoint(); iPoint++) - sol_c->GetNodes()->SetLocalCFL(iPoint, warmupCFL); - } - END_SU2_OMP_SAFE_GLOBAL_ACCESS - - } /*--- Set the current finest grid (full multigrid strategy) ---*/ FinestMesh = config[iZone]->GetFinestMesh(); - /*--- Perform the Full Approximation Scheme multigrid ---*/ - - MultiGrid_Cycle(geometry, solver_container, numerics_container, config, - FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); - - /*--- Adapt coarse-grid CFL once per cycle using smoothing residuals gathered during the cycle. ---*/ + /*--- Rebuild coarse-grid CFL before the cycle so the currently active FMG + * level uses the intended CFL in this iteration. ---*/ const unsigned short nMGLevels = config[iZone]->GetnMGLevels(); BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { - /*--- Use the current finest-grid CFL as the base for deterministic - * coarse-level scaling. Fall back to config scalar when local CFL - * adaptation is disabled. ---*/ + /*--- Use the level-0 flow CFL as the base reference and derive all coarse + * levels from it via MG_CFL_SCALING[i] = CFL(i+1)/CFL(i). Fall back to + * config scalar when local level-0 CFL is unavailable. ---*/ passivedouble cfl_base = SU2_TYPE::GetValue( - solver_container[iZone][iInst][FinestMesh][Solver_Position]->GetAvg_CFL_Local()); + solver_container[iZone][iInst][MESH_0][Solver_Position]->GetAvg_CFL_Local()); if (cfl_base < EPS) - cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)); + cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(MESH_0)); const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; passivedouble CFL_local = cfl_base; - for (unsigned short iMesh = FinestMesh; iMesh < nMGLevels; ++iMesh) { - const unsigned short lvl = iMesh + 1; - /*--- Use per-level scaling factor to increase coarse CFL (allows values > 1.0). - * Index into cflScaling is iMesh (0-based transition). ---*/ - const passivedouble scale = (iMesh < cflScaling.size()) - ? max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[iMesh])) + for (unsigned short lvl = 1; lvl <= nMGLevels; ++lvl) { + /*--- Index into cflScaling is (lvl-1): transition lvl-1 -> lvl. ---*/ + const unsigned short iScale = lvl - 1; + const passivedouble scale = (iScale < cflScaling.size()) + ? max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[iScale])) : passivedouble{0.25}; CFL_local *= scale; config[iZone]->SetCFL(lvl, CFL_local); @@ -269,17 +238,22 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, } END_SU2_OMP_SAFE_GLOBAL_ACCESS - /*--- Propagate the updated coarse-grid CFL to every coarse-grid point (all threads). ---*/ - for (unsigned short iMesh = FinestMesh; iMesh < nMGLevels; ++iMesh) { - const passivedouble CFL_coarse_new = SU2_TYPE::GetValue(config[iZone]->GetCFL(iMesh+1)); - CGeometry* geo_c = geometry[iZone][iInst][iMesh+1]; - CSolver* sol_c = solver_container[iZone][iInst][iMesh+1][Solver_Position]; + /*--- Propagate updated CFL to all coarse-grid points before the cycle. ---*/ + for (unsigned short iMesh = 1; iMesh <= nMGLevels; ++iMesh) { + const passivedouble CFL_coarse_new = SU2_TYPE::GetValue(config[iZone]->GetCFL(iMesh)); + CGeometry* geo_c = geometry[iZone][iInst][iMesh]; + CSolver* sol_c = solver_container[iZone][iInst][iMesh][Solver_Position]; SU2_OMP_FOR_STAT(roundUpDiv(geo_c->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geo_c->GetnPoint(); iPoint++) sol_c->GetNodes()->SetLocalCFL(iPoint, CFL_coarse_new); END_SU2_OMP_FOR } + /*--- Perform the Full Approximation Scheme multigrid ---*/ + + MultiGrid_Cycle(geometry, solver_container, numerics_container, config, + FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); + /*--- Computes primitive variables and gradients in the finest mesh (useful for the next solver (turbulence) and output ---*/ solver_container[iZone][iInst][MESH_0][Solver_Position]->Preprocessing(geometry[iZone][iInst][MESH_0], From 56134c50b822aa6580b6ba7c76c82c5ffda154b3 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 12:06:14 +0200 Subject: [PATCH 14/54] fix cfl for full multigrid --- SU2_CFD/include/solvers/CSolver.hpp | 10 ++++++++++ SU2_CFD/src/integration/CMultiGridIntegration.cpp | 1 + 2 files changed, 11 insertions(+) diff --git a/SU2_CFD/include/solvers/CSolver.hpp b/SU2_CFD/include/solvers/CSolver.hpp index 9d86e6afc5f..67b7a7be69b 100644 --- a/SU2_CFD/include/solvers/CSolver.hpp +++ b/SU2_CFD/include/solvers/CSolver.hpp @@ -395,6 +395,16 @@ class CSolver { */ inline void SetAvg_CFL_Local(su2double val_cfl) { Avg_CFL_Local = val_cfl; } + /*! + * \brief Set min/max/avg local CFL summary statistics. + * \param[in] val_cfl - Uniform CFL value to report. + */ + inline void SetCFL_Local_Stats(su2double val_cfl) { + Min_CFL_Local = val_cfl; + Max_CFL_Local = val_cfl; + Avg_CFL_Local = val_cfl; + } + /*! * \brief Get the number of variables of the problem. */ diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 1697356a817..8a721bf58f9 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -243,6 +243,7 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, const passivedouble CFL_coarse_new = SU2_TYPE::GetValue(config[iZone]->GetCFL(iMesh)); CGeometry* geo_c = geometry[iZone][iInst][iMesh]; CSolver* sol_c = solver_container[iZone][iInst][iMesh][Solver_Position]; + SU2_OMP_SAFE_GLOBAL_ACCESS(sol_c->SetCFL_Local_Stats(CFL_coarse_new);) SU2_OMP_FOR_STAT(roundUpDiv(geo_c->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geo_c->GetnPoint(); iPoint++) sol_c->GetNodes()->SetLocalCFL(iPoint, CFL_coarse_new); From 335e2c2bfc9174d8bf6e8f107a3fa51d3aa8d486 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 14:26:24 +0200 Subject: [PATCH 15/54] extra scaling for warmup phaser --- SU2_CFD/src/integration/CMultiGridIntegration.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 8a721bf58f9..e8d5198d522 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -223,6 +223,21 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, if (cfl_base < EPS) cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(MESH_0)); + /*--- FMG warmup: all warmup phases (FinestMesh > 0) solve a full problem from + * initial conditions on their active level; coarse-correction levels also + * use non-restricted ICs at first activation. Reduce the base CFL by + * 0.5^nMGLevels uniformly across all levels during warmup. This exactly + * replicates running with a reduced CFL_NUMBER during warmup and then + * recovering the full CFL when FinestMesh = 0 (V-cycle phase). + * Example (nMGLevels=2, CFL_NUMBER=100, MG_CFL_SCALING=0.3): + * Warmup phases 1 & 2: base=25, CFL[1]=7.5, CFL[2]=2.25 + * V-cycle phase: base=100, CFL[1]=30, CFL[2]=9 ---*/ + if (FinestMesh > 0) { + passivedouble warmup_factor = 1.0; + for (unsigned short k = 0; k < nMGLevels; ++k) warmup_factor *= 0.01; + cfl_base *= warmup_factor; + } + const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; passivedouble CFL_local = cfl_base; From 006ecd57b8f706925932686dd500dad9bb83f9cd Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 19:48:53 +0200 Subject: [PATCH 16/54] cleanup implicit lines --- Common/src/geometry/CMultiGridGeometry.cpp | 99 ------------------- .../src/integration/CMultiGridIntegration.cpp | 15 --- .../integration/CSingleGridIntegration.cpp | 17 ++++ 3 files changed, 17 insertions(+), 114 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index feeb1e2c60a..4936ae4bdbb 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1485,34 +1485,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (lines.empty()) return; - if (rank == MASTER_NODE) { - cout << "Implicit line agglomeration: detected " << lines.size() << " lines." << endl; - cout << " Mode: " << (ISOTROPIC ? "ISOTROPIC" : "ANISOTROPIC") << endl; - /*--- Show line length distribution ---*/ - size_t min_len = ULONG_MAX, max_len = 0; - su2double avg_len = 0.0; - for (const auto& L : lines) { - min_len = min(min_len, L.size()); - max_len = max(max_len, L.size()); - avg_len += L.size(); - } - if (!lines.empty()) avg_len /= lines.size(); - cout << " Line lengths: min=" << min_len << ", max=" << max_len << ", avg=" << std::setprecision(1) << std::fixed - << avg_len << endl; - - /*--- Show first few lines for debugging ---*/ - cout << " First 5 lines (showing first 4 nodes):" << endl; - for (size_t i = 0; i < min(size_t(5), lines.size()); ++i) { - cout << " Line " << i << " (len=" << lines[i].size() << "): ["; - for (size_t j = 0; j < min(size_t(4), lines[i].size()); ++j) { - if (j > 0) cout << ", "; - cout << lines[i][j]; - } - if (lines[i].size() > 4) cout << ", ..."; - cout << "]" << endl; - } - } - /*--- Agglomeration strategy: * ANISOTROPIC (default): Pair nodes at the SAME distance from wall on DIFFERENT lines. * Each coarse CV has 2 fine children (from adjacent lines). @@ -1607,16 +1579,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, nodes->SetChildren_CV(Index_CoarseCV, 3, d); nodes->SetnChildren_CV(Index_CoarseCV, 4); - /*--- Debug output: show CV creation details ---*/ - if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { - const auto* coord_a = fine_grid->nodes->GetCoord(a); - const auto* coord_b = fine_grid->nodes->GetCoord(b); - cout << " CV " << Index_CoarseCV << " (ISO): nodes " << a << "+" << b << "+" << c << "+" << d << " | lines[" - << li1 << "][" << idx1 << "," << idx2 << "]+lines[" << li2_best << "][" << idx1 << "," << idx2 << "]" - << " | coord_a=(" << coord_a[0] << "," << coord_a[1] << ")" - << " coord_b=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; - } - reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; MGQueue_InnerCV.RemoveCV(a); MGQueue_InnerCV.RemoveCV(b); @@ -1677,29 +1639,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- Geometrical quality check ---*/ if (!GeometricalCheck(b, fine_grid, config)) continue; - /*--- Debug: Check line distance and neighbor relationships ---*/ - if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { - /*--- Measure distance between wall vertices of the two lines ---*/ - const auto wall_a = lines[li1][0]; - const auto wall_b = lines[li2_best][0]; - const auto* coord_wall_a = fine_grid->nodes->GetCoord(wall_a); - const auto* coord_wall_b = fine_grid->nodes->GetCoord(wall_b); - su2double wall_dist = - sqrt(pow(coord_wall_a[0] - coord_wall_b[0], 2) + pow(coord_wall_a[1] - coord_wall_b[1], 2)); - - /*--- Check if wall vertices are neighbors ---*/ - bool walls_are_neighbors = false; - for (auto neighbor : fine_grid->nodes->GetPoints(wall_a)) { - if (neighbor == wall_b) { - walls_are_neighbors = true; - break; - } - } - - cout << " Pairing lines " << li1 << " + " << li2_best << " at pos=" << pos << " | wall_dist=" << wall_dist - << " | walls_neighbors=" << (walls_are_neighbors ? "YES" : "NO") << endl; - } - /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); nodes->SetChildren_CV(Index_CoarseCV, 0, a); @@ -1707,25 +1646,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, nodes->SetChildren_CV(Index_CoarseCV, 1, b); nodes->SetnChildren_CV(Index_CoarseCV, 2); - /*--- Debug output: show CV creation details ---*/ - if (DEBUG_OUTPUT && Index_CoarseCV < starting_Index_CoarseCV + DEBUG_CV_LIMIT) { - const auto* coord_a = fine_grid->nodes->GetCoord(a); - const auto* coord_b = fine_grid->nodes->GetCoord(b); - su2double dist = sqrt(pow(coord_a[0] - coord_b[0], 2) + pow(coord_a[1] - coord_b[1], 2)); - bool are_neighbors = false; - for (auto neighbor : fine_grid->nodes->GetPoints(a)) { - if (neighbor == b) { - are_neighbors = true; - break; - } - } - cout << " CV " << Index_CoarseCV << " (ANISO): nodes " << a << "+" << b << " | lines[" << li1 << "][" << pos - << "]+lines[" << li2_best << "][" << pos << "]" - << " | dist=" << dist << " | neighbors=" << (are_neighbors ? "YES" : "NO") << " | coords A=(" - << coord_a[0] << "," << coord_a[1] << ")" - << " B=(" << coord_b[0] << "," << coord_b[1] << ")" << endl; - } - reserved[a] = reserved[b] = 1; MGQueue_InnerCV.RemoveCV(a); MGQueue_InnerCV.RemoveCV(b); @@ -1784,25 +1704,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (nNodes_unpaired > 0) { cout << " WARNING: " << nNodes_unpaired << " nodes on implicit lines were left unpaired!" << endl; cout << " These will be processed by domain agglomeration (may create wrong orientation)." << endl; - - /*--- Show first few unpaired nodes ---*/ - unsigned long count = 0; - for (size_t li = 0; li < lines.size() && count < 10; ++li) { - const auto& L = lines[li]; - for (size_t i = 1; i < L.size() && count < 10; ++i) { - if (!reserved[L[i]]) { - cout << " Unpaired: line " << li << " node " << L[i] << " at position " << i << endl; - count++; - } - } - } - } - if (ISOTROPIC) { - cout << " Expected ratio: ~4 nodes per CV (actual: " << std::setprecision(2) << std::fixed - << (nCVs_created > 0 ? su2double(nNodes_claimed) / su2double(nCVs_created) : 0.0) << ")" << endl; - } else { - cout << " Expected ratio: ~2 nodes per CV (actual: " << std::setprecision(2) << std::fixed - << (nCVs_created > 0 ? su2double(nNodes_claimed) / su2double(nCVs_created) : 0.0) << ")" << endl; } } diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index e8d5198d522..8a721bf58f9 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -223,21 +223,6 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, if (cfl_base < EPS) cfl_base = SU2_TYPE::GetValue(config[iZone]->GetCFL(MESH_0)); - /*--- FMG warmup: all warmup phases (FinestMesh > 0) solve a full problem from - * initial conditions on their active level; coarse-correction levels also - * use non-restricted ICs at first activation. Reduce the base CFL by - * 0.5^nMGLevels uniformly across all levels during warmup. This exactly - * replicates running with a reduced CFL_NUMBER during warmup and then - * recovering the full CFL when FinestMesh = 0 (V-cycle phase). - * Example (nMGLevels=2, CFL_NUMBER=100, MG_CFL_SCALING=0.3): - * Warmup phases 1 & 2: base=25, CFL[1]=7.5, CFL[2]=2.25 - * V-cycle phase: base=100, CFL[1]=30, CFL[2]=9 ---*/ - if (FinestMesh > 0) { - passivedouble warmup_factor = 1.0; - for (unsigned short k = 0; k < nMGLevels; ++k) warmup_factor *= 0.01; - cfl_base *= warmup_factor; - } - const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; passivedouble CFL_local = cfl_base; diff --git a/SU2_CFD/src/integration/CSingleGridIntegration.cpp b/SU2_CFD/src/integration/CSingleGridIntegration.cpp index fab97842048..2806873d186 100644 --- a/SU2_CFD/src/integration/CSingleGridIntegration.cpp +++ b/SU2_CFD/src/integration/CSingleGridIntegration.cpp @@ -49,6 +49,23 @@ void CSingleGridIntegration::SingleGrid_Iteration(CGeometry ****geometry, CSolve CGeometry* geometry_fine = geometry[iZone][iInst][FinestMesh]; CSolver** solvers_fine = solver_container[iZone][iInst][FinestMesh]; + if (RunTime_EqSystem == RUNTIME_TURB_SYS) { + /*--- config[iZone]->GetCFL(FinestMesh) is already scaled for the active MG level + * by CMultiGridIntegration's per-level CFL rebuild (chain of MG_CFL_SCALING + * factors down to FinestMesh), including during FMG warmup. Do not re-apply + * that scaling here or it gets squared during warmup. ---*/ + const su2double turbReduction = SU2_TYPE::GetValue(config[iZone]->GetCFLRedCoeff_Turb()); + const su2double turbCFL = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)) * turbReduction; + auto* turbSolver = solvers_fine[Solver_Position]; + + SU2_OMP_SAFE_GLOBAL_ACCESS(turbSolver->SetCFL_Local_Stats(turbCFL);) + SU2_OMP_FOR_STAT(roundUpDiv(geometry_fine->GetnPoint(), omp_get_num_threads())) + for (auto iPoint = 0ul; iPoint < geometry_fine->GetnPoint(); ++iPoint) { + turbSolver->GetNodes()->SetLocalCFL(iPoint, turbCFL); + } + END_SU2_OMP_FOR + } + /*--- Preprocessing ---*/ solvers_fine[Solver_Position]->Preprocessing(geometry_fine, solvers_fine, config[iZone], From 1b84b72d76b90e68eeb19e6584fddb91dd07e1e2 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 19:59:05 +0200 Subject: [PATCH 17/54] cleanup implicit lines with claude --- Common/src/geometry/CMultiGridGeometry.cpp | 268 ++++++++------------- 1 file changed, 94 insertions(+), 174 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 4936ae4bdbb..9749e1c76d6 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1393,7 +1393,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const unsigned long nPointFine = fine_grid->GetnPoint(); const unsigned long starting_Index_CoarseCV = Index_CoarseCV; /*--- Track how many CVs we create ---*/ const bool DEBUG_OUTPUT = (rank == MASTER_NODE); /*--- Enable detailed diagnostic output ---*/ - const unsigned long DEBUG_CV_LIMIT = 20; /*--- Show details for first N CVs ---*/ /*--- Collect implicit lines starting at viscous (no-slip) wall vertices only. * Seeding from non-wall boundaries (farfield, inlet, outlet, symmetry) would @@ -1432,7 +1431,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, norm_prev = sqrt(norm_prev); for (unsigned short d = 0; d < nDim; ++d) prev_dir[d] /= norm_prev; - /*--- Build the implicit line by following the best-aligned interior neighbor ---*/ + /*--- Build the implicit line by following the best-aligned interior neighbor. + * Nodes already on this line are excluded from candidacy so the walk + * cannot fold back on itself (the angle threshold alone would only make + * that unlikely, not impossible, on distorted meshes). ---*/ vector L; L.push_back(iPoint); auto current = iPoint; @@ -1442,10 +1444,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long best_neighbor = ULONG_MAX; for (auto jPoint : fine_grid->nodes->GetPoints(current)) { - if (jPoint == current) continue; if (!fine_grid->nodes->GetDomain(jPoint)) continue; if (fine_grid->nodes->GetBoundary(jPoint)) continue; if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; + if (find(L.begin(), L.end(), jPoint) != L.end()) continue; /*--- Compute normalized direction to candidate ---*/ su2double vec[MAXNDIM] = {0.0}; @@ -1485,198 +1487,115 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (lines.empty()) return; - /*--- Agglomeration strategy: - * ANISOTROPIC (default): Pair nodes at the SAME distance from wall on DIFFERENT lines. - * Each coarse CV has 2 fine children (from adjacent lines). - * Reduces mesh by factor ~2 normal to wall, preserves resolution along wall. + /*--- Agglomeration strategy: at each "position" (distance from the wall along a + * line) every line contributes a block of nBlock consecutive nodes; two lines + * are paired through a mesh-adjacency search of their block anchors and the + * combined blocks become the children of one coarse CV. * - * ISOTROPIC: Group 4 nodes (2 positions × 2 lines) into one coarse CV. - * Each coarse CV has 4 fine children. - * Reduces mesh uniformly by factor ~4 in all directions. - ---*/ + * ANISOTROPIC (default, nBlock=1): pair single nodes at the SAME distance from + * the wall on two DIFFERENT lines. Each coarse CV has 2 fine children. + * Reduces the mesh by a factor ~2 normal to the wall, preserves resolution + * along the wall. + * + * ISOTROPIC (nBlock=2): pair 2-node blocks (2 positions x 2 lines) into one + * coarse CV with 4 fine children. Reduces the mesh uniformly by a factor + * ~4 in all directions. ---*/ + const unsigned long nBlock = ISOTROPIC ? 2 : 1; vector reserved(nPointFine, 0); - unsigned position_idx = 0; + unsigned long position_idx = 0; - while (true) { - bool any_work = false; - vector line_processed(lines.size(), 0); + /*--- Fine-grid nodes line `li` contributes at the current position, or empty if + the line is too short to reach that far. ---*/ + auto LineBlock = [&](unsigned long li) -> vector { + const auto& L = lines[li]; + const unsigned long first = 1 + nBlock * position_idx; + if (L.size() < first + nBlock) return {}; + return vector(L.begin() + first, L.begin() + first + nBlock); + }; - /*--- Build list of active lines (have nodes at current position) ---*/ + while (true) { + /*--- Cache each line's block for this position and collect the active ones. ---*/ + vector> block(lines.size()); vector active_lines; active_lines.reserve(lines.size()); for (unsigned long li = 0; li < lines.size(); ++li) { - const auto& L = lines[li]; - if (L.empty()) continue; - if (ISOTROPIC) { - const auto idx2 = 1 + 2 * position_idx + 1; - if (L.size() <= idx2) continue; // no pair at this stage - } else { - if (L.size() <= 1 + position_idx) continue; // no position at this index - } - active_lines.push_back(li); + block[li] = LineBlock(li); + if (!block[li].empty()) active_lines.push_back(li); } + if (active_lines.empty()) break; - if (ISOTROPIC) { - /*--- ISOTROPIC MODE: Group 4 children per coarse CV (2 positions × 2 lines) - Use spatial neighbor search to pair adjacent lines. ---*/ - for (auto li1 : active_lines) { - if (line_processed[li1]) continue; - - const auto& L1 = lines[li1]; - const auto idx1 = 1 + 2 * position_idx; - const auto idx2 = idx1 + 1; - if (L1.size() <= idx2) continue; - - const auto a = L1[idx1], b = L1[idx2]; - if (fine_grid->nodes->GetAgglomerate(a) || fine_grid->nodes->GetAgglomerate(b)) continue; - if (reserved[a] || reserved[b]) continue; - - /*--- Find nearest neighbor line by checking mesh neighbors of node 'a' ---*/ - unsigned long li2_best = std::numeric_limits::max(); - for (auto neighbor_point : fine_grid->nodes->GetPoints(a)) { - /*--- Check if this neighbor belongs to another unprocessed line at same position ---*/ - for (auto li2 : active_lines) { - if (li2 == li1 || line_processed[li2]) continue; - const auto& L2 = lines[li2]; - if (L2.size() <= idx2) continue; - const auto c = L2[idx1]; - if (c == neighbor_point) { - li2_best = li2; - break; - } - } - if (li2_best != std::numeric_limits::max()) break; - } - - if (li2_best == std::numeric_limits::max()) continue; - - const auto& L2 = lines[li2_best]; - const auto c = L2[idx1], d = L2[idx2]; - - /*--- Skip if any node is already claimed ---*/ - if (fine_grid->nodes->GetAgglomerate(c) || fine_grid->nodes->GetAgglomerate(d)) continue; - if (reserved[c] || reserved[d]) continue; - - /*--- Geometrical quality check ---*/ - if (!GeometricalCheck(a, fine_grid, config) || !GeometricalCheck(b, fine_grid, config) || - !GeometricalCheck(c, fine_grid, config) || !GeometricalCheck(d, fine_grid, config)) - continue; + vector line_processed(lines.size(), 0); + bool any_work = false; - /*--- Guard against duplicate indices ---*/ - if (a == b || a == c || a == d || b == c || b == d || c == d) { - line_processed[li1] = line_processed[li2_best] = 1; - continue; - } + for (auto li1 : active_lines) { + if (line_processed[li1]) continue; - /*--- Create 4-child coarse CV (isotropic agglomeration) ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - fine_grid->nodes->SetParent_CV(c, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 2, c); - fine_grid->nodes->SetParent_CV(d, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 3, d); - nodes->SetnChildren_CV(Index_CoarseCV, 4); - - reserved[a] = reserved[b] = reserved[c] = reserved[d] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - MGQueue_InnerCV.RemoveCV(c); - MGQueue_InnerCV.RemoveCV(d); + const auto anchor = block[li1].front(); + if (fine_grid->nodes->GetAgglomerate(anchor) || reserved[anchor]) continue; - Index_CoarseCV++; - line_processed[li1] = line_processed[li2_best] = 1; - any_work = true; - } - } else { - /*--- ANISOTROPIC MODE: Pair nodes at SAME position on DIFFERENT lines - Use spatial neighbor search to pair adjacent lines. ---*/ - for (auto li1 : active_lines) { - if (line_processed[li1]) continue; - - const auto& L1 = lines[li1]; - const auto pos = 1 + position_idx; - if (L1.size() <= pos) continue; - - const auto a = L1[pos]; - if (fine_grid->nodes->GetAgglomerate(a)) continue; - if (reserved[a]) continue; - if (!GeometricalCheck(a, fine_grid, config)) continue; - - /*--- Find nearest neighbor line by checking mesh neighbors of node 'a' ---*/ - unsigned long li2_best = std::numeric_limits::max(); - for (auto neighbor_point : fine_grid->nodes->GetPoints(a)) { - /*--- Check if this neighbor belongs to another unprocessed line at same position ---*/ - for (auto li2 : active_lines) { - if (li2 == li1 || line_processed[li2]) continue; - const auto& L2 = lines[li2]; - if (L2.size() <= pos) continue; - const auto b = L2[pos]; - if (b == neighbor_point) { - li2_best = li2; - break; - } + /*--- Find an unprocessed neighboring line: one whose block anchor is a + mesh-neighbor of ours at the same position. ---*/ + unsigned long li2 = ULONG_MAX; + for (auto neighbor_point : fine_grid->nodes->GetPoints(anchor)) { + for (auto candidate : active_lines) { + if (candidate == li1 || line_processed[candidate]) continue; + if (block[candidate].front() == neighbor_point) { + li2 = candidate; + break; } - if (li2_best != std::numeric_limits::max()) break; } + if (li2 != ULONG_MAX) break; + } - if (li2_best == std::numeric_limits::max()) { - /*--- Debug: Line couldn't find a neighbor ---*/ - if (DEBUG_OUTPUT && position_idx < 3) { - cout << " Line " << li1 << " at pos=" << pos << " (node " << a << ") has NO neighbor line!" << endl; - } - continue; + if (li2 == ULONG_MAX) { + if (DEBUG_OUTPUT && position_idx < 3) { + cout << " Line " << li1 << " at position " << position_idx << " (node " << anchor + << ") has NO neighbor line!" << endl; } - - const auto& L2 = lines[li2_best]; - const auto b = L2[pos]; - - /*--- Skip if partner is already claimed ---*/ - if (fine_grid->nodes->GetAgglomerate(b)) continue; - if (reserved[b]) continue; - - /*--- Geometrical quality check ---*/ - if (!GeometricalCheck(b, fine_grid, config)) continue; - - /*--- Create 2-child coarse CV (anisotropic: same position, different lines) ---*/ - fine_grid->nodes->SetParent_CV(a, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 0, a); - fine_grid->nodes->SetParent_CV(b, Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, 1, b); - nodes->SetnChildren_CV(Index_CoarseCV, 2); - - reserved[a] = reserved[b] = 1; - MGQueue_InnerCV.RemoveCV(a); - MGQueue_InnerCV.RemoveCV(b); - - Index_CoarseCV++; - line_processed[li1] = line_processed[li2_best] = 1; - any_work = true; + continue; } - } - position_idx++; - if (!any_work) break; + /*--- Assemble and validate the coarse CV's children together. ---*/ + auto group = block[li1]; + group.insert(group.end(), block[li2].begin(), block[li2].end()); - /*--- Check if any line still has positions available ---*/ - bool any_more = false; - if (ISOTROPIC) { - for (const auto& L : lines) { - if (L.size() > 1 + 2 * position_idx + 1) { - any_more = true; + bool valid = true; + for (auto p : group) { + if (fine_grid->nodes->GetAgglomerate(p) || reserved[p] || !GeometricalCheck(p, fine_grid, config)) { + valid = false; break; } } - } else { - for (const auto& L : lines) { - if (L.size() > 1 + position_idx) { - any_more = true; - break; - } + if (!valid) continue; // one of the nodes wasn't ready; li2 may still pair elsewhere. + + /*--- Guard against the same fine point appearing in both blocks + (can happen if two lines' walks overlap in space). ---*/ + bool duplicate = false; + for (size_t i = 0; !duplicate && i + 1 < group.size(); ++i) + for (size_t j = i + 1; j < group.size(); ++j) + if (group[i] == group[j]) duplicate = true; + + if (duplicate) { + line_processed[li1] = line_processed[li2] = 1; + continue; } + + /*--- Create the coarse CV from the combined blocks. ---*/ + for (unsigned short c = 0; c < group.size(); ++c) { + fine_grid->nodes->SetParent_CV(group[c], Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, c, group[c]); + reserved[group[c]] = 1; + MGQueue_InnerCV.RemoveCV(group[c]); + } + nodes->SetnChildren_CV(Index_CoarseCV, static_cast(group.size())); + Index_CoarseCV++; + + line_processed[li1] = line_processed[li2] = 1; + any_work = true; } - if (!any_more) break; + + if (!any_work) break; + position_idx++; } /*--- Count how many CVs and nodes were created ---*/ @@ -1696,7 +1615,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (reserved[i]) nNodes_claimed++; } - if (rank == MASTER_NODE) { + if (DEBUG_OUTPUT) { cout << " Created " << nCVs_created << " coarse CVs from " << nNodes_claimed << " fine nodes." << endl; cout << " Nodes on implicit lines: " << nNodes_on_lines << " (paired=" << (nNodes_on_lines - nNodes_unpaired) << ", unpaired=" << nNodes_unpaired << ")" << endl; @@ -1707,14 +1626,15 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } - /*--- Verify all claimed nodes are properly marked as agglomerated ---*/ + /*--- Verify all claimed nodes are properly marked as agglomerated (SetParent_CV should + guarantee this; a mismatch would indicate a bookkeeping bug above). ---*/ unsigned long mismatches = 0; for (unsigned long i = 0; i < nPointFine; ++i) { if (reserved[i] && !fine_grid->nodes->GetAgglomerate(i)) { mismatches++; } } - if (mismatches > 0 && rank == MASTER_NODE) { + if (mismatches > 0 && DEBUG_OUTPUT) { cout << " WARNING: " << mismatches << " nodes marked as reserved but not agglomerated!" << endl; } } From 2c1edf08cc96b8039eebedd2903a7ec184cd51aa Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 20:00:29 +0200 Subject: [PATCH 18/54] cleanup implicit lines with claude --- .../src/integration/CMultiGridIntegration.cpp | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 8a721bf58f9..57c2b32cc7a 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -59,33 +59,6 @@ static su2double applyGlobalTrend(su2double factor, passivedouble crossCycleRati return max(su2double{CLAMP_MIN}, min(su2double{CLAMP_MAX}, factor)); } -static su2double GetMGLevelCorrectionScale(unsigned short iMesh) { - switch (iMesh) { - case 0: return 1.00; - case 1: return 0.75; - case 2: return 0.50; - default: return 0.35; - } -} - -static passivedouble GetWarmupCFLScale(const std::vector& cflScaling, - unsigned long innerIter, - unsigned long startupIter) { - if (startupIter == 0 || cflScaling.empty()) return 1.0; - - const passivedouble firstScale = max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling.front())); - if (innerIter < startupIter) { - if (cflScaling.size() > 1) { - const passivedouble secondScale = max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[1])); - return firstScale * secondScale; - } - return firstScale; - } - - if (innerIter < 2 * startupIter) return firstScale; - return 1.0; -} - inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { passivedouble result = 0; for (unsigned short iVar = 0; iVar < solver->GetnVar(); ++iVar) { From 38b9c11b75a1d7c79820b3adfe091bf00c43c23b Mon Sep 17 00:00:00 2001 From: Nijso Date: Wed, 5 Aug 2026 20:34:48 +0200 Subject: [PATCH 19/54] Apply suggestion from @bigfooted --- SU2_CFD/src/integration/CMultiGridIntegration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 57c2b32cc7a..e34fd48149d 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -827,7 +827,7 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet for (auto iVar = 0u; iVar < nVar; iVar++) { /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ if (Residual_Fine[iVar] != Residual_Fine[iVar]) -Residual_Fine[iVar] = 0.0; + Residual_Fine[iVar] = 0.0; su2double correction = factor * Residual_Fine[iVar]; Solution_Fine[iVar] += correction; From e9f8b7cd71cedd9157160edec5b6690498f758ea Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 20:44:01 +0200 Subject: [PATCH 20/54] remove debugging output --- Common/src/geometry/CMultiGridGeometry.cpp | 106 +----------------- .../src/integration/CMultiGridIntegration.cpp | 4 +- .../integration/CSingleGridIntegration.cpp | 5 +- SU2_CFD/src/iteration/CFluidIteration.cpp | 2 +- 4 files changed, 6 insertions(+), 111 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 9749e1c76d6..7cea6d7d1c1 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -431,46 +431,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nPointDomain = Index_CoarseCV; nPoint = nPointDomain; - /*--- DIAGNOSTIC: Check CV child counts after domain agglomeration ---*/ - if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { - unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; - unsigned long n_corrupted_implicit_CVs = 0; - for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { - const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) - nCVs_1child++; - else if (nChildren == 2) - nCVs_2child++; - else if (nChildren == 3) - nCVs_3child++; - else if (nChildren == 4) - nCVs_4child++; - else - nCVs_other++; - - if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { - n_corrupted_implicit_CVs++; - if (n_corrupted_implicit_CVs <= 5) { - cout << " CORRUPTION DETECTED in CV " << iCV << ": has " << nChildren << " children (expected 2)" << endl; - cout << " Children nodes: "; - for (unsigned short iChild = 0; iChild < nChildren; iChild++) { - cout << nodes->GetChildren_CV(iCV, iChild); - if (iChild < nChildren - 1) cout << ", "; - } - cout << endl; - } - } - } - if (n_corrupted_implicit_CVs > 0) { - cout << " AFTER DOMAIN AGGLOMERATION: " << n_corrupted_implicit_CVs - << " implicit line CVs were corrupted (child count != 2)" << endl; - cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child - << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; - if (nCVs_other > 0) cout << ", other=" << nCVs_other; - cout << endl; - } - } - /*--- Check that there are no hanging nodes. Detect isolated points (only 1 neighbor), and merge their children CV's with the neighbor. ---*/ @@ -544,67 +504,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Diagnostic: Check if implicit line CVs were corrupted by hanging node correction ---*/ - if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { - unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; - unsigned long n_corrupted_after_hanging = 0; - for (auto iCV = Index_CoarseCV_before_implicit_lines; iCV < Index_CoarseCV_after_implicit_lines; iCV++) { - const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) - nCVs_1child++; - else if (nChildren == 2) - nCVs_2child++; - else if (nChildren == 3) - nCVs_3child++; - else if (nChildren == 4) - nCVs_4child++; - else - nCVs_other++; - - if (nChildren != 2 && !config->GetMGOptions().MG_Implicit_Lines_Isotropic) { - n_corrupted_after_hanging++; - } - } - if (n_corrupted_after_hanging > 0) { - cout << " AFTER HANGING NODE CORRECTION: " << n_corrupted_after_hanging - << " implicit line CVs corrupted (child count != 2)" << endl; - cout << " Distribution in implicit line CVs: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child - << ", 3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child; - if (nCVs_other > 0) cout << ", other=" << nCVs_other; - cout << endl; - } - } - - /*--- Final summary of all CVs ---*/ - if (config->GetMGOptions().MG_Implicit_Lines && (rank == MASTER_NODE)) { - cout << " Expected ratio: ~2 nodes per CV (actual: " << fixed << setprecision(2) - << (double)fine_grid->GetnPoint() / (double)nPointDomain << ")" << endl; - - unsigned long nCVs_1child = 0, nCVs_2child = 0, nCVs_3child = 0, nCVs_4child = 0, nCVs_other = 0; - for (auto iCV = 0ul; iCV < nPointDomain; iCV++) { - const auto nChildren = nodes->GetnChildren_CV(iCV); - if (nChildren == 1) - nCVs_1child++; - else if (nChildren == 2) - nCVs_2child++; - else if (nChildren == 3) - nCVs_3child++; - else if (nChildren == 4) - nCVs_4child++; - else - nCVs_other++; - } - cout << " CV distribution: 1-child=" << nCVs_1child << ", 2-child=" << nCVs_2child << ", 3-child=" << nCVs_3child - << ", 4-child=" << nCVs_4child; - if (nCVs_other > 0) cout << ", other=" << nCVs_other; - cout << endl; - - if (nCVs_3child > 0 || (!config->GetMGOptions().MG_Implicit_Lines_Isotropic && nCVs_4child > 0)) { - cout << " WARNING: Detected unexpected CV child counts (3-child=" << nCVs_3child << ", 4-child=" << nCVs_4child - << " in ANISO mode)" << endl; - } - } - /*--- Reset the neighbor information. ---*/ nodes->ResetPoints(); @@ -1431,10 +1330,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, norm_prev = sqrt(norm_prev); for (unsigned short d = 0; d < nDim; ++d) prev_dir[d] /= norm_prev; - /*--- Build the implicit line by following the best-aligned interior neighbor. - * Nodes already on this line are excluded from candidacy so the walk - * cannot fold back on itself (the angle threshold alone would only make - * that unlikely, not impossible, on distorted meshes). ---*/ + /*--- Build the implicit line by following the best-aligned interior neighbor. ---*/ vector L; L.push_back(iPoint); auto current = iPoint; diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 57c2b32cc7a..f49674927cf 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -425,7 +425,6 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, solver_coarse->Preprocessing(geometry_coarse, solver_container_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem, false); - Space_Integration(geometry_coarse, solver_container_coarse, numerics_coarse, config, iMesh+1, NO_RK_ITER, RunTime_EqSystem); /*--- Compute $P_(k+1) = I^(k+1)_k(r_k) - r_(k+1) ---*/ @@ -452,6 +451,9 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, iMesh+1, nextRecurseParam, RunTime_EqSystem, iZone, iInst); } + /*--- Compute prolongated solution, and smooth the correction $u^(new)_k = u_k + + Smooth(I^k_(k+1)(u_(k+1)-I^(k+1)_k u_k))$ ---*/ + GetProlongated_Correction(RunTime_EqSystem, solver_fine, solver_coarse, geometry_fine, geometry_coarse, config); const auto& mgOpts = config->GetMGOptions(); diff --git a/SU2_CFD/src/integration/CSingleGridIntegration.cpp b/SU2_CFD/src/integration/CSingleGridIntegration.cpp index 2806873d186..0f051fdc2b6 100644 --- a/SU2_CFD/src/integration/CSingleGridIntegration.cpp +++ b/SU2_CFD/src/integration/CSingleGridIntegration.cpp @@ -50,10 +50,7 @@ void CSingleGridIntegration::SingleGrid_Iteration(CGeometry ****geometry, CSolve CSolver** solvers_fine = solver_container[iZone][iInst][FinestMesh]; if (RunTime_EqSystem == RUNTIME_TURB_SYS) { - /*--- config[iZone]->GetCFL(FinestMesh) is already scaled for the active MG level - * by CMultiGridIntegration's per-level CFL rebuild (chain of MG_CFL_SCALING - * factors down to FinestMesh), including during FMG warmup. Do not re-apply - * that scaling here or it gets squared during warmup. ---*/ + /*--- CFL scaling of turbulence during the warmup phase if FMG. ---*/ const su2double turbReduction = SU2_TYPE::GetValue(config[iZone]->GetCFLRedCoeff_Turb()); const su2double turbCFL = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)) * turbReduction; auto* turbSolver = solvers_fine[Solver_Position]; diff --git a/SU2_CFD/src/iteration/CFluidIteration.cpp b/SU2_CFD/src/iteration/CFluidIteration.cpp index 264effbdf7a..a79a9f1f8f4 100644 --- a/SU2_CFD/src/iteration/CFluidIteration.cpp +++ b/SU2_CFD/src/iteration/CFluidIteration.cpp @@ -98,7 +98,7 @@ void CFluidIteration::Iterate(COutput* output, CIntegration**** integration, CGe config[val_iZone]->SetGlobalParam(main_solver, RUNTIME_TURB_SYS); integration[val_iZone][val_iInst][TURB_SOL]->SingleGrid_Iteration(geometry, solver, numerics, config, - RUNTIME_TURB_SYS, val_iZone, val_iInst); + RUNTIME_TURB_SYS, val_iZone, val_iInst); } if (config[val_iZone]->GetKind_Species_Model() != SPECIES_MODEL::NONE) { From 05e68d3dcde4c638fbde04ac3b33e5e36449b616 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 5 Aug 2026 22:22:50 +0200 Subject: [PATCH 21/54] remove unused variable --- Common/src/geometry/CMultiGridGeometry.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 7cea6d7d1c1..871d6fa8197 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -312,11 +312,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } /*--- Agglomerate high-aspect-ratio interior nodes along implicit lines from walls. ---*/ - unsigned long Index_CoarseCV_before_implicit_lines = Index_CoarseCV; - unsigned long Index_CoarseCV_after_implicit_lines = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, MGQueue_InnerCV); - Index_CoarseCV_after_implicit_lines = Index_CoarseCV; } /*--- STEP 2: Agglomerate the domain points. ---*/ From f7677dcca4a71b16c61c4a8f91e108f68beb2933 Mon Sep 17 00:00:00 2001 From: Nijso Date: Wed, 5 Aug 2026 23:04:22 +0200 Subject: [PATCH 22/54] Potential fix for pull request finding 'CodeQL / Comparison of narrow type with wide type in loop condition' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- Common/src/geometry/CMultiGridGeometry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 871d6fa8197..b97d919385b 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1474,7 +1474,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } /*--- Create the coarse CV from the combined blocks. ---*/ - for (unsigned short c = 0; c < group.size(); ++c) { + for (size_t c = 0; c < group.size(); ++c) { fine_grid->nodes->SetParent_CV(group[c], Index_CoarseCV); nodes->SetChildren_CV(Index_CoarseCV, c, group[c]); reserved[group[c]] = 1; From ab003cb15ecf1305c40f274d2d233553ba4b1561 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Thu, 6 Aug 2026 13:47:09 +0200 Subject: [PATCH 23/54] update euler tests --- TestCases/euler/CRM/inv_CRM_JST.cfg | 16 ++++++++++------ TestCases/euler/channel/inv_channel.cfg | 9 ++++++--- TestCases/euler/naca0012/inv_NACA0012.cfg | 16 ++++++++++------ TestCases/euler/oneram6/inv_ONERAM6.cfg | 8 ++++---- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/TestCases/euler/CRM/inv_CRM_JST.cfg b/TestCases/euler/CRM/inv_CRM_JST.cfg index 63c749021f2..60080a0254b 100644 --- a/TestCases/euler/CRM/inv_CRM_JST.cfg +++ b/TestCases/euler/CRM/inv_CRM_JST.cfg @@ -43,11 +43,11 @@ MARKER_MONITORING= ( fuselage , Wing , HTP ) % ------------- COMMON PARAMETERS TO DEFINE THE NUMERICAL METHOD --------------% % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES -CFL_NUMBER= 5.0 +CFL_NUMBER= 100.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) -EXT_ITER= 99999 +ITER= 1000 LINEAR_SOLVER= FGMRES LINEAR_SOLVER_ERROR= 1E-1 LINEAR_SOLVER_ITER= 3 @@ -55,12 +55,16 @@ LINEAR_SOLVER_ITER= 3 % -------------------------- MULTIGRID PARAMETERS -----------------------------% % MGLEVEL= 3 +MG_MIN_MESHSIZE= 100 MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_SMOOTH_OUTPUT= YES +MG_SMOOTH_EARLY_EXIT= YES +MG_PRE_SMOOTH= ( 5, 5, 5, 5 ) +MG_POST_SMOOTH= ( 5, 5, 5, 5 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 +MG_CFL_SCALING= 0.5, 0.5, 0.5 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/euler/channel/inv_channel.cfg b/TestCases/euler/channel/inv_channel.cfg index 5161a04413d..8145521b147 100644 --- a/TestCases/euler/channel/inv_channel.cfg +++ b/TestCases/euler/channel/inv_channel.cfg @@ -47,11 +47,11 @@ MARKER_MONITORING= ( upper_wall, lower_wall ) % ------------- COMMON PARAMETERS DEFINING THE NUMERICAL METHOD ---------------% % NUM_METHOD_GRAD= GREEN_GAUSS -CFL_NUMBER= 6.0 +CFL_NUMBER= 100.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) -EXT_ITER= 999999 +ITER= 1000 % ------------------------ LINEAR SOLVER DEFINITION ---------------------------% % @@ -63,12 +63,15 @@ LINEAR_SOLVER_ITER= 3 % -------------------------- MULTIGRID PARAMETERS -----------------------------% % MGLEVEL= 3 -MGCYCLE= V_CYCLE +MGCYCLE= W_CYCLE +MG_SMOOTH_OUTPUT= NO +MG_SMOOTH_EARLY_EXIT= YES MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) MG_POST_SMOOTH= ( 4, 4, 4, 4 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) MG_DAMP_RESTRICTION= 0.5 MG_DAMP_PROLONGATION= 0.5 +MG_CFL_SCALING= 0.5, 0.5, 0.5 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/euler/naca0012/inv_NACA0012.cfg b/TestCases/euler/naca0012/inv_NACA0012.cfg index ca235bb07a3..814649dbc30 100644 --- a/TestCases/euler/naca0012/inv_NACA0012.cfg +++ b/TestCases/euler/naca0012/inv_NACA0012.cfg @@ -46,7 +46,7 @@ MARKER_DESIGNING = ( airfoil ) % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES OBJECTIVE_FUNCTION= DRAG -CFL_NUMBER= 5.0 +CFL_NUMBER= 100.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) ITER= 250 @@ -62,11 +62,15 @@ LINEAR_SOLVER_ITER= 5 % MGLEVEL= 3 MGCYCLE= W_CYCLE -MG_PRE_SMOOTH= ( 1, 2, 3, 3 ) -MG_POST_SMOOTH= ( 0, 0, 0, 0 ) -MG_CORRECTION_SMOOTH= ( 0, 0, 0, 0 ) -MG_DAMP_RESTRICTION= 1.0 -MG_DAMP_PROLONGATION= 1.0 +MG_SMOOTH_OUTPUT= YES +MG_SMOOTH_EARLY_EXIT= YES + +MG_PRE_SMOOTH= ( 5, 5, 5, 5 ) +MG_POST_SMOOTH= ( 5, 5, 5, 5 ) +MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) +MG_DAMP_RESTRICTION= 0.5 +MG_DAMP_PROLONGATION= 0.5 +MG_CFL_SCALING= 0.5, 0.5, 0.5 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/euler/oneram6/inv_ONERAM6.cfg b/TestCases/euler/oneram6/inv_ONERAM6.cfg index 6517b957871..59c19e14a53 100644 --- a/TestCases/euler/oneram6/inv_ONERAM6.cfg +++ b/TestCases/euler/oneram6/inv_ONERAM6.cfg @@ -49,15 +49,15 @@ MARKER_DESIGNING = ( WING ) % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES OBJECTIVE_FUNCTION= DRAG -CFL_NUMBER= 5.0 +CFL_NUMBER= 100.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) ITER= 99999 LINEAR_SOLVER= FGMRES LINEAR_SOLVER_PREC= LU_SGS -LINEAR_SOLVER_ERROR= 1E-6 -LINEAR_SOLVER_ITER= 2 +LINEAR_SOLVER_ERROR= 1E-1 +LINEAR_SOLVER_ITER= 5 % ----------------------- SLOPE LIMITER DEFINITION ----------------------------% % @@ -92,7 +92,7 @@ TIME_DISCRE_ADJFLOW= EULER_IMPLICIT % --------------------------- CONVERGENCE PARAMETERS --------------------------& % CONV_RESIDUAL_MINVAL= -12 -CONV_STARTITER= 25 +CONV_STARTITER= 10 CONV_CAUCHY_ELEMS= 100 CONV_CAUCHY_EPS= 1E-10 From f5b52a0f652221322cee5fe7f9fd8b71922df507 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 9 Aug 2026 09:48:13 +0200 Subject: [PATCH 24/54] fix mg update, remove linear prolongation for now --- .../integration/CMultiGridIntegration.hpp | 21 ++ .../src/integration/CMultiGridIntegration.cpp | 264 ++++++++++++++++-- 2 files changed, 263 insertions(+), 22 deletions(-) diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 0672d1ae5ff..21c06a2a75f 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -26,6 +26,7 @@ */ #include "CIntegration.hpp" +#include "../../../Common/include/containers/container_decorators.hpp" /*! * \class CMultiGridIntegration @@ -277,4 +278,24 @@ class CMultiGridIntegration final : public CIntegration { unsigned short lastPreSmoothWorstStep[MAX_MG_LEVELS+1] = {}; unsigned short lastPostSmoothWorstStep[MAX_MG_LEVELS+1] = {}; + /*--- FMG startup CFL ramp bookkeeping: tracks the currently active FMG level + * and the InnerIter at which it became active, so its CFL can be ramped + * linearly towards the next (finer) level's target over MG_Startup_Iter + * iterations instead of jumping discontinuously at promotion. ---*/ + unsigned short mg_ramp_last_FinestMesh = MAX_MG_LEVELS + 1; /*!< \brief FinestMesh observed on the previous call; sentinel forces a reset on the first call. */ + unsigned long mg_ramp_level_start_iter = 0; /*!< \brief InnerIter at which the currently active FMG level became active. */ + + /*--- User-configured damping factors, captured before any adaptation so they can be + * restored whenever the active FMG level changes (the cross-cycle EMA that drives + * the adaptation is only meaningful within a single level). ---*/ + bool mg_damp_initial_captured = false; /*!< \brief Whether the configured damping factors have been stored yet. */ + su2double mg_damp_restric_initial = 0.0; /*!< \brief MG_DAMP_RESTRICTION as configured. */ + su2double mg_damp_prolong_initial = 0.0; /*!< \brief MG_DAMP_PROLONGATION as configured. */ + + /*--- FMG startup convergence-based early exit: promote the active level as soon as its + * CONV_FIELD residual has dropped two orders of magnitude, instead of always waiting + * out the full MG_Startup_Iter budget. Reset whenever the active level changes. ---*/ + passivedouble mg_conv_field_start_rms = -1.0; /*!< \brief CONV_FIELD RMS at the start of the active level's window; <0 = not yet captured. */ + bool mg_conv_field_early_exit = false; /*!< \brief Set once the active level has converged two orders of magnitude; consumed at the next promotion check. */ + }; diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 39b380a0df3..d5df808a7aa 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -26,6 +26,7 @@ */ #include "../../include/integration/CMultiGridIntegration.hpp" +#include "../../include/gradients/computeGradientsGreenGauss.hpp" #include "../../../Common/include/parallelization/omp_structure.hpp" #include "../../../Common/include/toolboxes/printing_toolbox.hpp" #include @@ -67,6 +68,53 @@ inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { return sqrt(result); } +/*!\cond PRIVATE + * Resolve the RMS residual named by CONV_FIELD (its first entry) for a given MG level, so the + * FMG startup early-exit criterion tracks the same quantity the user already monitors for + * overall convergence. Covers the RMS_* residual fields of the compressible and incompressible + * flow solvers, mirroring the field->variable mapping in CFlowCompOutput, + * CFlowIncOutput and CFlowOutput. CONV_FIELD entries that are not a residual (e.g. a force or + * Cauchy field) fall back to the primary flow residual (index 0), the same default those output + * classes use for RMS_DENSITY / RMS_PRESSURE. + \endcond */ +inline passivedouble ResolveConvFieldRMS(const CConfig* config, CSolver* const* solver_lvl, unsigned short nDim) { + const string field = (config->GetnConv_Field() > 0) ? config->GetConv_Field(0) : string("RMS_DENSITY"); + const CSolver* flow = solver_lvl[FLOW_SOL]; + + if (field == "RMS_DENSITY" || field == "RMS_PRESSURE") + return SU2_TYPE::GetValue(flow->GetRes_RMS(0)); + if (field == "RMS_MOMENTUM-X" || field == "RMS_VELOCITY-X") + return SU2_TYPE::GetValue(flow->GetRes_RMS(1)); + if (field == "RMS_MOMENTUM-Y" || field == "RMS_VELOCITY-Y") + return SU2_TYPE::GetValue(flow->GetRes_RMS(2)); + if (nDim == 3 && (field == "RMS_MOMENTUM-Z" || field == "RMS_VELOCITY-Z")) + return SU2_TYPE::GetValue(flow->GetRes_RMS(3)); + if (field == "RMS_ENERGY") + return SU2_TYPE::GetValue(flow->GetRes_RMS(nDim + 1)); + + return SU2_TYPE::GetValue(flow->GetRes_RMS(0)); +} + +/*!\cond PRIVATE + * Prolongate a coarse-grid field onto the fine grid via constant injection: every fine + * child gets its parent's value. \c getCoarse returns the coarse-grid block of a point + * and \c setFine writes it to a fine-grid point, so the same loop serves both the FAS + * correction and the Full-MG solution handoff. + * + * Only domain points are written; halo synchronization is the caller's responsibility. + \endcond */ +template +void ProlongateField(CGeometry* geo_coarse, GetCoarse getCoarse, SetFine setFine) { + + SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) + for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { + for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { + auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); + setFine(Point_Fine, getCoarse(Point_Coarse)); + } + } + END_SU2_OMP_FOR + } // anonymous namespace void CMultiGridIntegration::adaptDampingFactors(CConfig* config, passivedouble crossCycleRatio) { @@ -159,12 +207,25 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, END_SU2_OMP_SAFE_GLOBAL_ACCESS /*--- Full MG: advance to the next finer grid after a fixed number of - * outer iterations on the current coarsest active level. - * The number of iterations per level is controlled by MG_STARTUP_ITER config option. ---*/ + * outer iterations on the current coarsest active level, controlled by + * MG_STARTUP_ITER, or as soon as the level's CONV_FIELD residual has already + * dropped two orders of magnitude (mg_conv_field_early_exit, set below after + * the cycle runs) - whichever happens first. A level that converges quickly + * should not sit idle for the rest of its budget before being promoted. + * + * Iterations spent on the currently active level are counted relative to + * mg_ramp_level_start_iter (the InnerIter at which it became active, reset on + * every promotion below) rather than a global InnerIter modulo. A global modulo + * assumes every level consumes exactly MG_STARTUP_ITER iterations; once a level + * is promoted early via mg_conv_field_early_exit, InnerIter falls out of phase + * with that assumption and every subsequent level's fixed-iteration budget would + * be truncated to whatever remains until the next global phase boundary instead + * of getting its own full MG_STARTUP_ITER window. ---*/ const unsigned long startup_iter = config[iZone]->GetMGOptions().MG_Startup_Iter; + const unsigned long iters_on_level = config[iZone]->GetInnerIter() - mg_ramp_level_start_iter; const bool Convergence_FullMG = FullMG && (FinestMesh != MESH_0) && - (config[iZone]->GetInnerIter() % startup_iter == startup_iter - 1); + ((iters_on_level >= startup_iter - 1) || mg_conv_field_early_exit); if (!config[iZone]->GetRestart() && FullMG && direct && ( Convergence_FullMG && (FinestMesh != MESH_0 )) && RunTime_EqSystem == RUNTIME_FLOW_SYS) { @@ -176,6 +237,21 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, geometry[iZone][iInst][FinestMesh], config[iZone]); + /*--- Report the promotion and why it happened now, before mg_ramp_level_start_iter + * and mg_conv_field_early_exit are reset for the newly active level below. ---*/ + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS + if (SU2_MPI::GetRank() == MASTER_NODE) { + cout << "Full-MG: mesh level " << FinestMesh << " -> " << FinestMesh - 1 << " after " + << (iters_on_level + 1) << " iteration(s) ("; + if (mg_conv_field_early_exit) { + const string convField = (config[iZone]->GetnConv_Field() > 0) ? config[iZone]->GetConv_Field(0) : string("RMS_DENSITY"); + cout << convField << " dropped 2 orders of magnitude"; + } else + cout << "MG_STARTUP_ITER= " << startup_iter << " reached"; + cout << ")." << endl; + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS + SU2_OMP_SAFE_GLOBAL_ACCESS(config[iZone]->SubtractFinestMesh();) } @@ -188,9 +264,41 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, const unsigned short nMGLevels = config[iZone]->GetnMGLevels(); BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { - /*--- Use the level-0 flow CFL as the base reference and derive all coarse - * levels from it via MG_CFL_SCALING[i] = CFL(i+1)/CFL(i). Fall back to - * config scalar when local level-0 CFL is unavailable. ---*/ + /*--- Capture the configured damping factors once, before any adaptation has + * modified them, so they can be restored at every FMG level change. ---*/ + if (!mg_damp_initial_captured) { + mg_damp_restric_initial = config[iZone]->GetDamp_Res_Restric(); + mg_damp_prolong_initial = config[iZone]->GetDamp_Correc_Prolong(); + mg_damp_initial_captured = true; + } + + /*--- Detect a change of active FMG level (promotion, restart, or a new + * outer/time-step resetting FinestMesh) and (re)start the ramp window. ---*/ + if (FinestMesh != mg_ramp_last_FinestMesh) { + mg_ramp_level_start_iter = config[iZone]->GetInnerIter(); + mg_ramp_last_FinestMesh = FinestMesh; + + /*--- The cross-cycle EMA is only meaningful while the active level is fixed: + * after a promotion the pre-smoothing RMS is measured on a different grid, + * so the ratio reflects the change of level rather than the convergence + * trend. Left alone, the lagging EMA drives both damping factors onto a + * clamp within ~20-30 cycles - towards CLAMP_MIN, crippling the correction, + * or towards CLAMP_MAX, making it maximally aggressive on a freshly + * prolongated solution. Reseed the EMA and restore the configured factors. ---*/ + mg_fine_rms_ema = 0.0; + config[iZone]->SetDamp_Res_Restric(mg_damp_restric_initial); + config[iZone]->SetDamp_Correc_Prolong(mg_damp_prolong_initial); + + /*--- Restart the convergence-based early-exit window: the baseline is (re)captured + * below, after this iteration's cycle has produced a fresh residual on the level. ---*/ + mg_conv_field_start_rms = -1.0; + mg_conv_field_early_exit = false; + } + + /*--- Use the level-0 flow CFL as the base reference and derive the steady-state + * scaled target for all coarse levels from it via + * MG_CFL_SCALING[i] = CFL(i+1)/CFL(i). Fall back to config scalar when + * local level-0 CFL is unavailable. ---*/ passivedouble cfl_base = SU2_TYPE::GetValue( solver_container[iZone][iInst][MESH_0][Solver_Position]->GetAvg_CFL_Local()); if (cfl_base < EPS) @@ -198,14 +306,48 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, const auto& cflScaling = config[iZone]->GetMGOptions().MG_CflScaling; - passivedouble CFL_local = cfl_base; + passivedouble CFL_target[MAX_MG_LEVELS+1]; + passivedouble CFL_scale[MAX_MG_LEVELS+1]; + CFL_target[0] = cfl_base; + CFL_scale[0] = 1.0; for (unsigned short lvl = 1; lvl <= nMGLevels; ++lvl) { /*--- Index into cflScaling is (lvl-1): transition lvl-1 -> lvl. ---*/ const unsigned short iScale = lvl - 1; const passivedouble scale = (iScale < cflScaling.size()) ? max(passivedouble{1e-6}, SU2_TYPE::GetValue(cflScaling[iScale])) : passivedouble{0.25}; - CFL_local *= scale; + CFL_scale[lvl] = scale; + CFL_target[lvl] = CFL_target[lvl-1] * scale; + } + + /*--- During FMG startup, linearly ramp the CFL of the currently active level up + * to its own scaled target over MG_Startup_Iter iterations, starting from the + * CFL the previously active (coarser) level was running at. The ramp is + * therefore continuous across a promotion and, crucially, never exceeds the + * level's own target: the sustainable CFL is a property of the mesh, so + * carrying a finer level's target onto a coarser mesh destabilises it. The + * reduced starting value also gives the freshly prolongated solution time to + * relax before the level runs at full CFL. The coarsest level has no coarser + * predecessor, so it extends the scaling one step further to soften the + * initial transient away from the freestream state. + * + * All non-active coarse levels keep their steady-state scaled target, and no + * ramp is applied once FMG has reached MESH_0 (the final V-cycle-equivalent + * stage behaves exactly like a plain V-cycle). ---*/ + const bool ramping = FullMG && (FinestMesh > MESH_0) && (FinestMesh <= nMGLevels); + passivedouble ramp_progress = 1.0; + if (ramping && startup_iter > 0) { + const unsigned long iter_in_level = config[iZone]->GetInnerIter() - mg_ramp_level_start_iter; + ramp_progress = min(passivedouble{1.0}, passivedouble(iter_in_level) / passivedouble(startup_iter)); + } + + for (unsigned short lvl = 1; lvl <= nMGLevels; ++lvl) { + passivedouble CFL_local = CFL_target[lvl]; + if (ramping && lvl == FinestMesh) { + const passivedouble CFL_start = (lvl < nMGLevels) ? CFL_target[lvl+1] + : CFL_target[lvl] * CFL_scale[lvl]; + CFL_local = (passivedouble(1.0) - ramp_progress) * CFL_start + ramp_progress * CFL_target[lvl]; + } config[iZone]->SetCFL(lvl, CFL_local); } } @@ -228,6 +370,26 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, MultiGrid_Cycle(geometry, solver_container, numerics_container, config, FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); + /*--- FMG startup convergence-based early exit: track the active level's CONV_FIELD + * residual and flag promotion once it has dropped two orders of magnitude, so a + * level that converges well inside its MG_Startup_Iter budget is not held back. + * Reuses this same cycle's fresh residual, so the very first iteration of a window + * only seeds the baseline (nothing to compare against yet). ---*/ + if (FullMG && direct && (FinestMesh != MESH_0) && RunTime_EqSystem == RUNTIME_FLOW_SYS) { + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS + { + const unsigned short nDim = geometry[iZone][iInst][FinestMesh]->GetnDim(); + const passivedouble conv_now = ResolveConvFieldRMS(config[iZone], solver_container[iZone][iInst][FinestMesh], nDim); + + if (mg_conv_field_start_rms < 0.0) { + mg_conv_field_start_rms = max(conv_now, passivedouble(EPS)); + } else if (conv_now <= 1e-2 * mg_conv_field_start_rms) { + mg_conv_field_early_exit = true; + } + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS + } + /*--- Computes primitive variables and gradients in the finest mesh (useful for the next solver (turbulence) and output ---*/ solver_container[iZone][iInst][MESH_0][Solver_Position]->Preprocessing(geometry[iZone][iInst][MESH_0], @@ -723,14 +885,15 @@ void CMultiGridIntegration::GetProlongated_Correction(unsigned short RunTime_EqS sol_coarse->InitiateComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); sol_coarse->CompleteComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) - for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { - for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { - auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); - sol_fine->LinSysRes.SetBlock(Point_Fine, sol_coarse->GetNodes()->GetSolution_Old(Point_Coarse)); - } - } - END_SU2_OMP_FOR + /*--- Interpolate the coarse-grid correction (held in Solution_Old) onto the fine + * grid and store it in LinSysRes, which SetProlongated_Correction then damps + * and adds to the fine-grid solution. ---*/ + + ProlongateField(geo_coarse, + [&](unsigned long iPoint) { return sol_coarse->GetNodes()->GetSolution_Old(iPoint); }, + [&](unsigned long Point_Fine, const su2double* value) { + sol_fine->LinSysRes.SetBlock(Point_Fine, value); + }); } @@ -848,14 +1011,71 @@ void CMultiGridIntegration::SetProlongated_Solution(unsigned short RunTime_EqSys CGeometry *geo_fine, CGeometry *geo_coarse, CConfig *config) { SU2_ZONE_SCOPED - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) - for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { - for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { - auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); - sol_fine->GetNodes()->SetSolution(Point_Fine, sol_coarse->GetNodes()->GetSolution(Point_Coarse)); + const unsigned short Solver_Position = config->GetContainerPosition(RunTime_EqSystem); + const bool grid_movement = config->GetGrid_Movement(); + + /*--- Interpolate the coarse solution onto the fine grid. This is the initial + * condition the newly activated Full-MG level starts from, so it uses the same + * constant-injection operator as the FAS correction, applied directly to the + * solution rather than to LinSysRes. ---*/ + + ProlongateField(geo_coarse, + [&](unsigned long iPoint) { return sol_coarse->GetNodes()->GetSolution(iPoint); }, + [&](unsigned long Point_Fine, const su2double* value) { + sol_fine->GetNodes()->SetSolution(Point_Fine, value); + }); + + /*--- Update the solution at the no-slip walls, mirroring SetRestricted_Solution. + * The prolongated values come from coarse-grid nodes and do not satisfy the + * fine-grid wall conditions on their own. ---*/ + + for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { + if (config->GetViscous_Wall(iMarker)) { + + SU2_OMP_FOR_STAT(32) + for (auto iVertex = 0ul; iVertex < geo_fine->nVertex[iMarker]; iVertex++) { + + const auto Point_Fine = geo_fine->vertex[iMarker][iVertex]->GetNode(); + + if (Solver_Position == FLOW_SOL) { + + /*--- At moving walls, set the solution based on the new density and wall velocity ---*/ + + if (grid_movement) { + const auto* Grid_Vel = geo_fine->nodes->GetGridVel(Point_Fine); + sol_fine->GetNodes()->SetVelSolutionVector(Point_Fine, Grid_Vel); + } + else { + /*--- For stationary no-slip walls, set the velocity to zero. ---*/ + su2double zero[3] = {0.0}; + sol_fine->GetNodes()->SetVelSolutionVector(Point_Fine, zero); + } + + } + + if (Solver_Position == ADJFLOW_SOL) { + sol_fine->GetNodes()->SetVelSolutionDVector(Point_Fine); + } + + } + END_SU2_OMP_FOR } } - END_SU2_OMP_FOR + + /*--- Enforce Euler wall BC by projecting velocity to the tangent plane. The coarse + * velocity is tangent to the coarse wall normal, which is not the fine-grid wall + * normal on curved surfaces; without this the smallest near-wall cells start with + * a spurious normal velocity. ---*/ + + sol_fine->MultigridProjectEulerWall(geo_fine, config, false); + + /*--- MPI the new interpolated solution. The loops above only write domain points, + * while Preprocessing builds primitive variables over all points including halos, + * so the halos must be synchronized before the level is iterated. ---*/ + + sol_fine->InitiateComms(geo_fine, config, MPI_QUANTITIES::SOLUTION); + sol_fine->CompleteComms(geo_fine, config, MPI_QUANTITIES::SOLUTION); + } void CMultiGridIntegration::SetForcing_Term(CSolver *sol_fine, CSolver *sol_coarse, CGeometry *geo_fine, From 6792ff7214ab1876d5913dadd25687a608d4dadd Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 9 Aug 2026 09:59:17 +0200 Subject: [PATCH 25/54] cleanup early exit for FMG warmup phase --- .../integration/CMultiGridIntegration.hpp | 1 - .../src/integration/CMultiGridIntegration.cpp | 44 ++++--------------- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 21c06a2a75f..f7db01424e9 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -26,7 +26,6 @@ */ #include "CIntegration.hpp" -#include "../../../Common/include/containers/container_decorators.hpp" /*! * \class CMultiGridIntegration diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index d5df808a7aa..a6250c89de7 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -26,7 +26,6 @@ */ #include "../../include/integration/CMultiGridIntegration.hpp" -#include "../../include/gradients/computeGradientsGreenGauss.hpp" #include "../../../Common/include/parallelization/omp_structure.hpp" #include "../../../Common/include/toolboxes/printing_toolbox.hpp" #include @@ -68,33 +67,6 @@ inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { return sqrt(result); } -/*!\cond PRIVATE - * Resolve the RMS residual named by CONV_FIELD (its first entry) for a given MG level, so the - * FMG startup early-exit criterion tracks the same quantity the user already monitors for - * overall convergence. Covers the RMS_* residual fields of the compressible and incompressible - * flow solvers, mirroring the field->variable mapping in CFlowCompOutput, - * CFlowIncOutput and CFlowOutput. CONV_FIELD entries that are not a residual (e.g. a force or - * Cauchy field) fall back to the primary flow residual (index 0), the same default those output - * classes use for RMS_DENSITY / RMS_PRESSURE. - \endcond */ -inline passivedouble ResolveConvFieldRMS(const CConfig* config, CSolver* const* solver_lvl, unsigned short nDim) { - const string field = (config->GetnConv_Field() > 0) ? config->GetConv_Field(0) : string("RMS_DENSITY"); - const CSolver* flow = solver_lvl[FLOW_SOL]; - - if (field == "RMS_DENSITY" || field == "RMS_PRESSURE") - return SU2_TYPE::GetValue(flow->GetRes_RMS(0)); - if (field == "RMS_MOMENTUM-X" || field == "RMS_VELOCITY-X") - return SU2_TYPE::GetValue(flow->GetRes_RMS(1)); - if (field == "RMS_MOMENTUM-Y" || field == "RMS_VELOCITY-Y") - return SU2_TYPE::GetValue(flow->GetRes_RMS(2)); - if (nDim == 3 && (field == "RMS_MOMENTUM-Z" || field == "RMS_VELOCITY-Z")) - return SU2_TYPE::GetValue(flow->GetRes_RMS(3)); - if (field == "RMS_ENERGY") - return SU2_TYPE::GetValue(flow->GetRes_RMS(nDim + 1)); - - return SU2_TYPE::GetValue(flow->GetRes_RMS(0)); -} - /*!\cond PRIVATE * Prolongate a coarse-grid field onto the fine grid via constant injection: every fine * child gets its parent's value. \c getCoarse returns the coarse-grid block of a point @@ -114,6 +86,7 @@ void ProlongateField(CGeometry* geo_coarse, GetCoarse getCoarse, SetFine setFine } } END_SU2_OMP_FOR +} } // anonymous namespace @@ -370,16 +343,17 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, MultiGrid_Cycle(geometry, solver_container, numerics_container, config, FinestMesh, RecursiveParam, RunTime_EqSystem, iZone, iInst); - /*--- FMG startup convergence-based early exit: track the active level's CONV_FIELD - * residual and flag promotion once it has dropped two orders of magnitude, so a - * level that converges well inside its MG_Startup_Iter budget is not held back. - * Reuses this same cycle's fresh residual, so the very first iteration of a window - * only seeds the baseline (nothing to compare against yet). ---*/ + /*--- FMG startup convergence-based early exit: track the active level's aggregate flow + * residual (RMS across all solution variables, the same solver-agnostic metric this + * class already uses for the smoothing early-exit diagnostics) and flag promotion once + * it has dropped two orders of magnitude, so a level that converges well inside its + * MG_Startup_Iter budget is not held back. Reuses this same cycle's fresh residual, so + * the very first iteration of a window only seeds the baseline (nothing to compare + * against yet). ---*/ if (FullMG && direct && (FinestMesh != MESH_0) && RunTime_EqSystem == RUNTIME_FLOW_SYS) { BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { - const unsigned short nDim = geometry[iZone][iInst][FinestMesh]->GetnDim(); - const passivedouble conv_now = ResolveConvFieldRMS(config[iZone], solver_container[iZone][iInst][FinestMesh], nDim); + const passivedouble conv_now = ComputeLinSysResRMS(solver_container[iZone][iInst][FinestMesh][Solver_Position]); if (mg_conv_field_start_rms < 0.0) { mg_conv_field_start_rms = max(conv_now, passivedouble(EPS)); From 2a9fa0dcc0d83d600820feccf72a63aa557c57f6 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 9 Aug 2026 10:37:36 +0200 Subject: [PATCH 26/54] only show mg output when mglevel>0 --- SU2_CFD/src/integration/CMultiGridIntegration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index a6250c89de7..c8ca4fcdfae 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -400,8 +400,8 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, END_SU2_OMP_SAFE_GLOBAL_ACCESS } - /*--- Print compact smoothing summary when MG_SMOOTH_OUTPUT= YES. ---*/ - if (mgOptsZone.MG_Smooth_Output) { + /*--- Print compact smoothing summary when MG_SMOOTH_OUTPUT= YES and MGLEVEL > 0. ---*/ + if ((mgOptsZone.MG_Smooth_Output) && (nMGLevels > 0)) { BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS if (SU2_MPI::GetRank() == MASTER_NODE) { From 4d831a93b7e9e12b6d2aeafb9f89e8469529a81d Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 9 Aug 2026 13:32:52 +0200 Subject: [PATCH 27/54] some small optimizations --- .../integration/CMultiGridIntegration.hpp | 7 ++ .../src/integration/CMultiGridIntegration.cpp | 76 ++++++++++++------- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index f7db01424e9..93e7f3008b7 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -250,6 +250,13 @@ class CMultiGridIntegration final : public CIntegration { static constexpr int MAX_MG_LEVELS = 10; + /*--- Upper bound on nVar for the small per-point scratch arrays used by the restriction and + * prolongation loops, so they can live on the stack instead of being heap-allocated on every + * call. Must be >= the largest MAXNVAR of any variable class integrated by this class, + * currently CNEMOEulerVariable::MAXNVAR = 25. ---*/ + static constexpr unsigned short MAXNVAR = 25; + static constexpr unsigned short MAXNDIM = 3; + /*--- Early-exit smoothing state (shared across OMP threads via master write + barrier). ---*/ bool mg_early_exit_flag = false; /*!< \brief Shared flag for early exit across OMP threads. */ passivedouble mg_initial_smooth_rms = 0.0; /*!< \brief Initial RMS residual before current smoothing phase (FAS). */ diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index c8ca4fcdfae..f415770aabb 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -73,13 +73,19 @@ inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { * and \c setFine writes it to a fine-grid point, so the same loop serves both the FAS * correction and the Full-MG solution handoff. * - * Only domain points are written; halo synchronization is the caller's responsibility. + * The loop covers all coarse points, halos included. Halo coarse CVs own the fine halo points as + * children (CMultiGridGeometry sets Children_CV for received CVs), so injecting from them is what + * fills the fine-grid halo entries of the prolongated field. Restricting the loop to domain points + * leaves those entries at whatever the last solver update left there (zero, for LinSysRes), which + * is wrong for any operator that reads the prolongated field at neighbours across a partition + * boundary - the Jacobi smoother in SmoothProlongated_Correction does exactly that. The caller + * must therefore have synchronized the coarse-grid field being read before calling this. \endcond */ template void ProlongateField(CGeometry* geo_coarse, GetCoarse getCoarse, SetFine setFine) { - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) - for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { + SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPoint(), omp_get_num_threads())) + for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPoint(); Point_Coarse++) { for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); setFine(Point_Fine, getCoarse(Point_Coarse)); @@ -573,6 +579,19 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, SU2_OMP_SAFE_GLOBAL_ACCESS(config->SetKind_TimeIntScheme(EULER_IMPLICIT);) } + /*--- NOTE: the coarse-grid residual computed just above is evaluated at the restricted + * solution, i.e. at exactly the state the first pre-smoothing sweep of the recursive call + * below re-evaluates it at, so it looks like that sweep could reuse LinSysRes (and, if the + * Jacobian were assembled here, the Jacobian too) and skip its own Preprocessing and + * Space_Integration. It cannot, as things stand: Space_Integration is not a pure producer of + * LinSysRes/Jacobian. BC_Sym_Plane (which serves both SYMMETRY_PLANE and EULER_WALL) also + * projects Res_TruncError and Solution_Old onto the wall tangent plane, and in the current + * ordering that projection is what makes the FAS forcing term written by SetForcing_Term + * below, and the Solution_Old written by Set_OldSolution, wall-consistent before their first + * use. Reusing the residual moves both projections to the wrong side of the writes. + * Factoring those side effects out of Space_Integration would make the reuse safe and save + * one full residual evaluation per coarse level per cycle. ---*/ + /*--- Recursive call to MultiGrid_Cycle (this routine). ---*/ /*--- Execute multigrid cycles sequentially to ensure deterministic recursion order ---*/ /*--- This prevents accumulation of floating-point variations across recursive calls ---*/ @@ -800,14 +819,13 @@ void CMultiGridIntegration::GetProlongated_Correction(unsigned short RunTime_EqS SU2_ZONE_SCOPED const unsigned short nVar = sol_coarse->GetnVar(); - su2activevector Solution(nVar); SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { - su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); + su2double Solution[MAXNVAR] = {0.0}; - Solution = su2double(0); + su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); /*--- Accumulate children contributions with stable ordering ---*/ /*--- Process all children in sequential order to ensure deterministic FP summation ---*/ @@ -826,8 +844,7 @@ void CMultiGridIntegration::GetProlongated_Correction(unsigned short RunTime_EqS for (auto iVar = 0u; iVar < nVar; iVar++) Solution[iVar] += Solution_Coarse[iVar]; - for (auto iVar = 0u; iVar < nVar; iVar++) - sol_coarse->GetNodes()->SetSolution_Old(Point_Coarse, Solution.data()); + sol_coarse->GetNodes()->SetSolution_Old(Point_Coarse, Solution); } END_SU2_OMP_FOR @@ -854,7 +871,9 @@ void CMultiGridIntegration::GetProlongated_Correction(unsigned short RunTime_EqS } } - /*--- MPI the set solution old ---*/ + /*--- MPI the set solution old. Required: the loop above only writes domain points, and + * ProlongateField below injects from every coarse point including halos in order to fill the + * fine-grid halo entries of the correction. ---*/ sol_coarse->InitiateComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); sol_coarse->CompleteComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); @@ -924,7 +943,15 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } END_SU2_OMP_FOR - /*--- Restore original residuals (without average) at boundary points. ---*/ + /*--- Restore original residuals (without average) at boundary points. + * + * FIXME (MPI): SEND_RECEIVE is not excluded here, so every point on a partition interface + * has its smoothed correction reverted after each sweep. That is why this smoother gives a + * different convergence history on 1 and on N ranks. Excluding SEND_RECEIVE is only half the + * fix: the sweeps also read LinSysRes at halo points, which ProlongateField fills once but + * nothing refreshes between sweeps, so a halo exchange of LinSysRes is needed inside the + * loop (there is no MPI_QUANTITIES entry for it yet). Until both are done this smoother is + * only parallel-consistent for MG_CORRECTION_SMOOTH= 0, which is the default. ---*/ for (auto iMarker = 0u; iMarker < geometry->GetnMarker(); iMarker++) { if ((config->GetMarker_All_KindBC(iMarker) != INTERNAL_BOUNDARY) && @@ -1056,26 +1083,23 @@ void CMultiGridIntegration::SetForcing_Term(CSolver *sol_fine, CSolver *sol_coar CGeometry *geo_coarse, CConfig *config, unsigned short iMesh) { SU2_ZONE_SCOPED - const su2double *Residual_Fine; - const unsigned short nVar = sol_coarse->GetnVar(); const su2double factor = config->GetDamp_Res_Restric(); - su2activevector RestrictedDefect(nVar); - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPointDomain(); Point_Coarse++) { sol_coarse->GetNodes()->SetRes_TruncErrorZero(Point_Coarse); - RestrictedDefect = su2double(0); + su2double RestrictedDefect[MAXNVAR] = {0.0}; + for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { auto Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); - Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); + const su2double* Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); for (auto iVar = 0u; iVar < nVar; iVar++) RestrictedDefect[iVar] += factor * Residual_Fine[iVar]; } - sol_coarse->GetNodes()->AddRes_TruncError(Point_Coarse, RestrictedDefect.data()); + sol_coarse->GetNodes()->AddRes_TruncError(Point_Coarse, RestrictedDefect); } END_SU2_OMP_FOR @@ -1174,17 +1198,15 @@ void CMultiGridIntegration::SetRestricted_Gradient(unsigned short RunTime_EqSyst const unsigned short nDim = geo_coarse->GetnDim(); const unsigned short nVar = sol_coarse->GetnVar(); - auto **Gradient = new su2double* [nVar]; - for (auto iVar = 0u; iVar < nVar; iVar++) - Gradient[iVar] = new su2double [nDim]; - SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPoint(), omp_get_num_threads())) for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPoint(); Point_Coarse++) { - su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); - for (auto iVar = 0u; iVar < nVar; iVar++) - for (auto iDim = 0u; iDim < nDim; iDim++) - Gradient[iVar][iDim] = 0.0; + /*--- Row-major scratch plus the row pointers SetGradient expects. ---*/ + su2double GradientData[MAXNVAR][MAXNDIM] = {{0.0}}; + su2double* Gradient[MAXNVAR]; + for (auto iVar = 0u; iVar < nVar; iVar++) Gradient[iVar] = GradientData[iVar]; + + su2double Area_Parent = geo_coarse->nodes->GetVolume(Point_Coarse); for (auto iChildren = 0u; iChildren < geo_coarse->nodes->GetnChildren_CV(Point_Coarse); iChildren++) { unsigned long Point_Fine = geo_coarse->nodes->GetChildren_CV(Point_Coarse, iChildren); @@ -1199,10 +1221,6 @@ void CMultiGridIntegration::SetRestricted_Gradient(unsigned short RunTime_EqSyst } END_SU2_OMP_FOR - for (auto iVar = 0u; iVar < nVar; iVar++) - delete [] Gradient[iVar]; - delete [] Gradient; - } void CMultiGridIntegration::NonDimensional_Parameters(CGeometry **geometry, CSolver ***solver_container, From 0ce67e7aeecfdfd90dc8230af4c54219b6aeb60c Mon Sep 17 00:00:00 2001 From: bigfooted Date: Tue, 11 Aug 2026 17:30:34 +0200 Subject: [PATCH 28/54] freeze preconditioner, 10 percent speed gain. --- Common/include/linear_algebra/CSysSolve.hpp | 7 +++++ Common/include/option_structure.hpp | 1 + Common/src/CConfig.cpp | 4 +++ Common/src/linear_algebra/CSysSolve.cpp | 29 ++++++++++++++++++++- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Common/include/linear_algebra/CSysSolve.hpp b/Common/include/linear_algebra/CSysSolve.hpp index 1f9bc851b92..337701c87b5 100644 --- a/Common/include/linear_algebra/CSysSolve.hpp +++ b/Common/include/linear_algebra/CSysSolve.hpp @@ -136,6 +136,13 @@ class CSysSolve { /*!< \brief Inner solver for nested preconditioning. */ std::unique_ptr> inner_solver; + /*--- Preconditioner freezing on coarse multigrid levels. The factorization lives in the + * CSysMatrix (not in the short-lived CPreconditioner object built in Solve), so simply + * skipping Build() reuses the previous one. This instance belongs to one solver on one + * grid level, so the counter is naturally per-level. See MG_COARSE_PREC_FREEZE. ---*/ + mutable unsigned long precSolveCount = 0; /*!< \brief Linear solves done by this instance. */ + mutable bool buildPrecThisSolve = true; /*!< \brief Decision for the current solve, shared by all threads. */ + /*! * \brief sign transfer function * \param[in] x - value having sign prescribed diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 5bda4780d23..8d1a8f9d119 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1128,6 +1128,7 @@ struct CMGOptions { unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ + unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ }; /*! diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 2d68a229b27..d8974518802 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2063,6 +2063,10 @@ void CConfig::SetConfig_Options() { addDoubleOption("MG_SMOOTH_STAGNATION_TOL", MGOptions.MG_Smooth_StagnationTol, 0.99); /*!\brief MG_SMOOTH_COEFF\n DESCRIPTION: Smoothing coefficient for the correction prolongation Jacobi smoother. DEFAULT: 1.25 \ingroup Config*/ addDoubleOption("MG_SMOOTH_COEFF", MGOptions.MG_Smooth_Coeff, 1.25); + /*!\brief MG_COARSE_PREC_FREEZE\n DESCRIPTION: On multigrid levels above MESH_0, reuse the linear-solver preconditioner + * (e.g. the ILU factorization) for this many consecutive linear solves instead of rebuilding it every time. + * 1 reproduces the previous behaviour exactly. DEFAULT: 1 \ingroup Config*/ + addUnsignedLongOption("MG_COARSE_PREC_FREEZE", MGOptions.MG_Coarse_Prec_Freeze, 1); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level. Levels that would produce fewer CVs are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index dcd88c452c5..0274faa0f60 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -1435,6 +1435,33 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con const bool nested = SetupInnerSolver(KindSolver, config); + /*--- Decide whether to rebuild the preconditioner for this solve. + * + * On coarse multigrid levels the Jacobian changes little between smoothing sweeps, yet the + * factorization is rebuilt from scratch on every one of them. Profiling a V-cycle shows the + * ILU build is the single most expensive zone in the cycle, so MG_COARSE_PREC_FREEZE lets a + * factorization be reused for several consecutive solves. The factorization itself lives in + * the CSysMatrix, which outlives the CPreconditioner object created below, so not calling + * Build() is all that is needed to reuse it. + * + * Restricted to the standard solver mode and to levels above MESH_0: the fine grid drives the + * outer nonlinear convergence and is not worth degrading. The first solve on this instance + * always builds (count 0), which matters because the factorization is otherwise uninitialized. + * + * The decision is taken by one thread and read by all of them, because Build() is internally + * OpenMP-parallel and every thread must make the same choice. ---*/ + + BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { + unsigned long freeze = 1; + if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && + geometry->GetMGLevel() != MESH_0) { + freeze = std::max(1, config->GetMGOptions().MG_Coarse_Prec_Freeze); + } + buildPrecThisSolve = (precSolveCount % freeze == 0); + precSolveCount++; + } + END_SU2_OMP_SAFE_GLOBAL_ACCESS + /*--- Stop the recording for the linear solver ---*/ bool TapeActive = NO; @@ -1472,7 +1499,7 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con const auto kindPrec = static_cast(KindPrecond); auto* normal_prec = CPreconditioner::Create(kindPrec, Jacobian, geometry, config); - normal_prec->Build(); + if (buildPrecThisSolve) normal_prec->Build(); CPreconditioner* nested_prec = nullptr; if (nested) { From 2c48c63a42d90266c8e5c0658f02756e0fbe1b81 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 14 Aug 2026 16:51:57 +0200 Subject: [PATCH 29/54] continued multigrid improvement --- Common/include/CConfig.hpp | 9 +- Common/include/option_structure.hpp | 3 + Common/include/option_structure.inl | 34 ++++-- Common/src/CConfig.cpp | 17 +++ Common/src/geometry/CGeometry.cpp | 103 +++++++++++++++--- Common/src/linear_algebra/CSysSolve.cpp | 15 ++- .../integration/CMultiGridIntegration.hpp | 6 + .../src/integration/CMultiGridIntegration.cpp | 68 +++++++++++- 8 files changed, 220 insertions(+), 35 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 1b2a45dc3ff..74a470733f8 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -639,7 +639,8 @@ class CConfig { su2double Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ su2double Deform_Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ su2double Linear_Solver_Smoother_Relaxation; /*!< \brief Relaxation factor for iterative linear smoothers. */ - unsigned long Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ + unsigned long Linear_Solver_Iter; + unsigned long Linear_Solver_Prec_Freeze; /*!< \brief Reuse the finest-grid preconditioner for this many solves. */ /*!< \brief Max iterations of the linear solver for the implicit formulation. */ unsigned long Deform_Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Frequency; /*!< \brief Restart frequency of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Deflation; /*!< \brief Number of vectors used for deflated restarts. */ @@ -4383,6 +4384,12 @@ class CConfig { */ unsigned long GetLinear_Solver_Iter(void) const { return Linear_Solver_Iter; } + /*! + * \brief Number of consecutive linear solves that reuse one finest-grid preconditioner. + * \return Freeze period, 1 meaning rebuild on every solve. + */ + unsigned long GetLinear_Solver_Prec_Freeze(void) const { return Linear_Solver_Prec_Freeze; } + /*! * \brief Get max number of iterations of the linear solver for the implicit formulation. * \return Max number of iterations of the linear solver for the implicit formulation. diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 8d1a8f9d119..f3af3d16446 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1129,6 +1129,9 @@ struct CMGOptions { bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ + su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ + su2double MG_Startup_Stagnation{0.0}; /*!< \brief FMG: promote when the residual ratio between successive iterations exceeds this. 0 = disabled. */ + unsigned long MG_Startup_Stagnation_Iter{5}; /*!< \brief FMG: consecutive stalled iterations required before promoting. */ }; /*! diff --git a/Common/include/option_structure.inl b/Common/include/option_structure.inl index cf85994db07..fe1b2df385e 100644 --- a/Common/include/option_structure.inl +++ b/Common/include/option_structure.inl @@ -1101,6 +1101,21 @@ struct CStringValuesListHelper { }; // Class where the option is represented by (string, N * "some type", string, N * "some type", ...) +/*! + * \brief Whether a config token is a numeric value rather than a name. + * + * Options that interleave marker names with numbers have to tell the two apart. Testing the first + * character for a letter is not enough: mesh formats such as CGNS routinely produce boundary names + * that begin with a digit (4000_QUAD_4_Bdy6), which such a test reads as a value. Requiring the + * whole token to parse as a number is unambiguous for every name that is not purely numeric. + */ +inline bool IsNumericToken(const std::string& token) { + if (token.empty()) return false; + char* end = nullptr; + std::strtod(token.c_str(), &end); + return (end != token.c_str()) && (*end == '\0'); +} + template class COptionStringValuesList final : public COptionBase { const string name; // identifier for the option @@ -1143,15 +1158,20 @@ class COptionStringValuesList final : public COptionBase { return ""; } - /*--- Determine the number of strings: A new string is found if the first char in the option is a letter. - * This will fail in if a string starts with a number! Additionally, determine the number of values that - * are prescribed per string. ---*/ + /*--- Determine the number of strings: a field that does not parse as a number starts a new string, + * anything that does is one of its values. Testing only the first character for a letter would + * misread the digit-leading marker names that CGNS meshes produce. Additionally, determine the + * number of values that are prescribed per string. ---*/ vector num_vals_per_string; /*--- Loop through the fields of the option. ---*/ for (const auto& val : option_value) { - if (isalpha(val[0])) { + if (!IsNumericToken(val)) { num_vals_per_string.push_back(0); } else { + if (num_vals_per_string.empty()) + SU2_MPI::Error(name + string(" must begin with a marker name, but starts with the value \"") + val + + string("\". A marker whose name is purely numeric cannot be told apart from a value."), + CURRENT_FUNCTION); num_vals_per_string.back()++; } } @@ -1360,15 +1380,15 @@ class COptionWallSpecies : public COptionBase { /*--- Determine the number of markers and species per marker. * Format: marker1, TYPE1, value1, TYPE2, value2, ..., marker2, TYPE1, value1, ... - * Each marker name starts with a letter, each TYPE is an enum string (starts with letter), - * and each value is numeric. Pattern: marker, (TYPE, value) x N ---*/ + * Marker names and TYPE keywords are non-numeric fields, values are numeric. + * Pattern: marker, (TYPE, value) x N ---*/ vector marker_indices; // Indices where markers start vector species_counts; // Number of species per marker // Find all marker positions (strings starting with a letter that are not TYPE keywords) for (unsigned short i = 0; i < totalVals; i++) { - if (isalpha(option_value[i][0])) { + if (!IsNumericToken(option_value[i])) { // Check if this could be a TYPE keyword (i.e., is it in the enum map?) if (this->m.find(option_value[i]) != m.end()) { continue; // This is a TYPE keyword, not a marker diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index d8974518802..e0b53ff79ba 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2067,6 +2067,15 @@ void CConfig::SetConfig_Options() { * (e.g. the ILU factorization) for this many consecutive linear solves instead of rebuilding it every time. * 1 reproduces the previous behaviour exactly. DEFAULT: 1 \ingroup Config*/ addUnsignedLongOption("MG_COARSE_PREC_FREEZE", MGOptions.MG_Coarse_Prec_Freeze, 1); + /*!\brief LINEAR_SOLVER_PREC_FREEZE\n DESCRIPTION: Reuse the linear-solver preconditioner (e.g. the ILU factorization) + * for this many consecutive solves on the finest grid, instead of rebuilding it every time. Applies to MESH_0 and to + * single-grid runs; MG_COARSE_PREC_FREEZE covers the coarse levels. The fine-grid preconditioner drives the outer + * nonlinear convergence, so raise this one with more care. 1 rebuilds every solve. DEFAULT: 1 \ingroup Config*/ + addUnsignedLongOption("LINEAR_SOLVER_PREC_FREEZE", Linear_Solver_Prec_Freeze, 1); + /*!\brief MG_CORRECTION_LIMIT\n DESCRIPTION: Largest relative change any solution component may undergo from a single + * prolongated multigrid correction, e.g. 0.1 caps it at 10%. The whole correction vector at a point is scaled by one + * factor so its direction is preserved. 0 disables the limiter (previous behaviour). DEFAULT: 0 \ingroup Config*/ + addDoubleOption("MG_CORRECTION_LIMIT", MGOptions.MG_Correction_Limit, 0.0); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level. Levels that would produce fewer CVs are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ @@ -2077,6 +2086,14 @@ void CConfig::SetConfig_Options() { addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); + /*!\brief MG_STARTUP_STAGNATION\n DESCRIPTION: Full-MG promotion on stagnation. If the active level's residual ratio + * between successive iterations exceeds this value for MG_STARTUP_STAGNATION_ITER consecutive iterations, promote to + * the next finer level without waiting out MG_STARTUP_ITER. A coarse level is only worth iterating while it still + * reduces the error. 0 disables it. DEFAULT: 0.99 \ingroup Config*/ + addDoubleOption("MG_STARTUP_STAGNATION", MGOptions.MG_Startup_Stagnation, 0.99); + /*!\brief MG_STARTUP_STAGNATION_ITER\n DESCRIPTION: Consecutive stalled iterations required before Full-MG promotes + * on stagnation. DEFAULT: 5 \ingroup Config*/ + addUnsignedLongOption("MG_STARTUP_STAGNATION_ITER", MGOptions.MG_Startup_Stagnation_Iter, 5); /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); diff --git a/Common/src/geometry/CGeometry.cpp b/Common/src/geometry/CGeometry.cpp index 45153456086..a64e6c5db91 100644 --- a/Common/src/geometry/CGeometry.cpp +++ b/Common/src/geometry/CGeometry.cpp @@ -25,6 +25,7 @@ * License along with SU2. If not, see . */ +#include #include #include "../../include/geometry/CGeometry.hpp" @@ -4356,6 +4357,19 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) unsigned long maxNPoints = 0, sumNPoints = 0; + /*--- Why each line stopped growing. A line that ends because the mesh has become isotropic is a + * line that has done its job; one that ends because no neighbour was well enough aligned means + * the walk lost the wall-normal direction and the line is short despite the mesh still being + * stretched. The two call for opposite responses on coarse grids, so they are counted apart. ---*/ + unsigned long nStopIsotropic = 0, nStopNoNeighbour = 0, nStopCap = 0; + /*--- "No neighbour" has two very different causes: every candidate was already taken by another + * line (a competition/ordering problem), or candidates were free but none lay within 45 deg of + * the current direction (a geometry problem). Only the second says the mesh lost its + * wall-normal structure. For the latter, also accumulate the best alignment on offer, which + * says whether the 45 deg threshold is merely too tight or the direction is truly lost. ---*/ + unsigned long nStopAllTaken = 0, nStopMisaligned = 0; + su2double sumBestCos = 0.0; + if (nLinelet != 0) { /*--- Define the basic linelets, starting from each vertex, preventing duplication of points. ---*/ @@ -4375,11 +4389,23 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } li.linelets.resize(nLinelet); - /*--- Create the linelet structure. ---*/ + /*--- Grow the lines breadth first: each pass advances every still-growing line by one point. + * Growing them depth first - running one line to completion before starting the next - lets an + * early line exhaust its own column, turn sideways (the 45 deg test permits it) and consume the + * points its neighbours needed, starving them into one- and two-point stubs. That is harmless on + * the fine grid, where the boundary layer is deeper than MAX_LINELET_POINTS so no line ever runs + * out of vertical room, but on agglomerated grids the layer is shallower than the cap and the + * starvation is severe. Advancing in lockstep makes the lines compete on equal terms for the + * layer they are all entitled to. ---*/ - nLinelet = 0; - for (auto& linelet : li.linelets) { - while (linelet.size() < CLineletInfo::MAX_LINELET_POINTS) { + std::vector growing(nLinelet, 1); + + for (unsigned long step = 1; step < CLineletInfo::MAX_LINELET_POINTS; ++step) { + bool anyGrew = false; + + for (auto iLine = 0ul; iLine < nLinelet; ++iLine) { + if (!growing[iLine]) continue; + auto& linelet = li.linelets[iLine]; const auto iPoint = linelet.back(); /*--- Compute the value of the max and min weights to detect if this region is isotropic. ---*/ @@ -4398,26 +4424,35 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } /*--- Isotropic, stop this linelet. ---*/ - if (min_weight / max_weight > CLineletInfo::ALPHA_ISOTROPIC()) break; + if (min_weight / max_weight > CLineletInfo::ALPHA_ISOTROPIC()) { + growing[iLine] = 0; + ++nStopIsotropic; + continue; + } /*--- Otherwise, add the closest valid neighbor. ---*/ su2double min_dist2 = std::numeric_limits::max(); auto next_Point = iPoint; const auto* iCoord = nodes->GetCoord(iPoint); + unsigned long nFreeCandidates = 0; + su2double bestCos = -1.0; for (const auto jPoint : nodes->GetPoints(iPoint)) { if (li.lineletIdx[jPoint] == CLineletInfo::NO_LINELET && nodes->GetDomain(jPoint)) { + ++nFreeCandidates; const auto* jCoord = nodes->GetCoord(jPoint); const su2double d2 = GeometryToolbox::SquaredDistance(nDim, iCoord, jCoord); su2double cosTheta = 1; + su2double dij[3] = {0.0}; + GeometryToolbox::Distance(nDim, jCoord, iCoord, dij); if (linelet.size() > 1) { const auto* kCoord = nodes->GetCoord(linelet[linelet.size() - 2]); - su2double dij[3] = {0.0}, dki[3] = {0.0}; + su2double dki[3] = {0.0}; GeometryToolbox::Distance(nDim, iCoord, kCoord, dki); - GeometryToolbox::Distance(nDim, jCoord, iCoord, dij); cosTheta = GeometryToolbox::DotProduct(3, dki, dij) / sqrt(d2 * GeometryToolbox::SquaredNorm(nDim, dki)); } + bestCos = max(bestCos, cosTheta); if (d2 < min_dist2 && cosTheta > 0.7071) { next_Point = jPoint; min_dist2 = d2; @@ -4426,28 +4461,62 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } /*--- Did not find a suitable point. ---*/ - if (next_Point == iPoint) break; + if (next_Point == iPoint) { + growing[iLine] = 0; + ++nStopNoNeighbour; + if (nFreeCandidates == 0) { + ++nStopAllTaken; + } else { + ++nStopMisaligned; + sumBestCos += bestCos; + } + continue; + } linelet.push_back(next_Point); - li.lineletIdx[next_Point] = nLinelet; + li.lineletIdx[next_Point] = iLine; + anyGrew = true; } - ++nLinelet; - maxNPoints = max(maxNPoints, linelet.size()); - sumNPoints += linelet.size(); + if (!anyGrew) break; + } + + /*--- A line that never stopped advancing ran into the length cap. ---*/ + for (auto iLine = 0ul; iLine < nLinelet; ++iLine) { + if (growing[iLine]) ++nStopCap; + maxNPoints = max(maxNPoints, li.linelets[iLine].size()); + sumNPoints += li.linelets[iLine].size(); } } /*--- Average linelet size over all ranks. ---*/ - unsigned long globalNPoints, globalNLineLets; + unsigned long globalNPoints, globalNLineLets, globalMaxNPoints; + unsigned long stopCounts[5] = {nStopIsotropic, nStopNoNeighbour, nStopCap, nStopAllTaken, nStopMisaligned}; + unsigned long globalStop[5] = {}; SU2_MPI::Allreduce(&sumNPoints, &globalNPoints, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&nLinelet, &globalNLineLets, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - - if (rank == MASTER_NODE) { - std::cout << "Computed linelet structure, " + SU2_MPI::Allreduce(&maxNPoints, &globalMaxNPoints, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); + SU2_MPI::Allreduce(stopCounts, globalStop, 5, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + su2double globalSumBestCos = 0.0; + SU2_MPI::Allreduce(&sumBestCos, &globalSumBestCos, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); + + if (rank == MASTER_NODE && globalNLineLets > 0) { + const auto pct = [&](unsigned long n) { return 100.0 * passivedouble(n) / globalNLineLets; }; + std::cout << "Computed linelet structure on MG level " << MGLevel << ", " << static_cast(passivedouble(globalNPoints) / globalNLineLets) - << " points in each line (average)." << std::endl; + << " points in each line (average), " << globalMaxNPoints << " longest, " + << globalNLineLets << " lines.\n" + << " Line ends because: " << pct(globalStop[0]) << "% mesh became isotropic, " + << pct(globalStop[1]) << "% no aligned neighbour, " + << pct(globalStop[2]) << "% hit the " << CLineletInfo::MAX_LINELET_POINTS + << "-point cap.\n" + << " of the 'no aligned neighbour': " << pct(globalStop[3]) + << "% all candidates already claimed by another line, " << pct(globalStop[4]) + << "% candidates free but misaligned"; + if (globalStop[4] > 0) + std::cout << " (best cos on offer " << globalSumBestCos / globalStop[4] << ", need > 0.7071)"; + std::cout << "." << std::endl; } /*--- Color the linelets for OpenMP parallelization and visualization. ---*/ diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index 0274faa0f60..41bb8bb6370 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -1444,18 +1444,21 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con * the CSysMatrix, which outlives the CPreconditioner object created below, so not calling * Build() is all that is needed to reuse it. * - * Restricted to the standard solver mode and to levels above MESH_0: the fine grid drives the - * outer nonlinear convergence and is not worth degrading. The first solve on this instance - * always builds (count 0), which matters because the factorization is otherwise uninitialized. + * Restricted to the standard solver mode (mesh deformation and gradient smoothing are left + * alone). Coarse levels and the finest grid have separate periods because the finest-grid + * preconditioner drives the outer nonlinear convergence and so carries more risk. The first + * solve on this instance always builds (count 0), which matters because the factorization is + * otherwise uninitialized. * * The decision is taken by one thread and read by all of them, because Build() is internally * OpenMP-parallel and every thread must make the same choice. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { unsigned long freeze = 1; - if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && - geometry->GetMGLevel() != MESH_0) { - freeze = std::max(1, config->GetMGOptions().MG_Coarse_Prec_Freeze); + if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr) { + freeze = (geometry->GetMGLevel() != MESH_0) ? config->GetMGOptions().MG_Coarse_Prec_Freeze + : config->GetLinear_Solver_Prec_Freeze(); + freeze = std::max(1, freeze); } buildPrecThisSolve = (precSolveCount % freeze == 0); precSolveCount++; diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 93e7f3008b7..2818026dd44 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -304,4 +304,10 @@ class CMultiGridIntegration final : public CIntegration { passivedouble mg_conv_field_start_rms = -1.0; /*!< \brief CONV_FIELD RMS at the start of the active level's window; <0 = not yet captured. */ bool mg_conv_field_early_exit = false; /*!< \brief Set once the active level has converged two orders of magnitude; consumed at the next promotion check. */ + /*--- FMG stagnation promotion: a coarse level is only worth iterating while it still reduces the + * error, which happens well before a fixed iteration budget expires on fine meshes. ---*/ + passivedouble mg_fmg_prev_rms = -1.0; /*!< \brief Active level's residual on the previous iteration; <0 = not yet captured. */ + unsigned long mg_fmg_stall_count = 0; /*!< \brief Consecutive iterations without useful reduction on the active level. */ + bool mg_fmg_promoted_on_stall = false; /*!< \brief Whether the pending promotion was triggered by stagnation (for reporting). */ + }; diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index f415770aabb..31512265df3 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -222,7 +222,10 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, if (SU2_MPI::GetRank() == MASTER_NODE) { cout << "Full-MG: mesh level " << FinestMesh << " -> " << FinestMesh - 1 << " after " << (iters_on_level + 1) << " iteration(s) ("; - if (mg_conv_field_early_exit) { + if (mg_fmg_promoted_on_stall) { + cout << "residual stalled for " << config[iZone]->GetMGOptions().MG_Startup_Stagnation_Iter + << " iteration(s)"; + } else if (mg_conv_field_early_exit) { const string convField = (config[iZone]->GetnConv_Field() > 0) ? config[iZone]->GetConv_Field(0) : string("RMS_DENSITY"); cout << convField << " dropped 2 orders of magnitude"; } else @@ -272,6 +275,9 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, * below, after this iteration's cycle has produced a fresh residual on the level. ---*/ mg_conv_field_start_rms = -1.0; mg_conv_field_early_exit = false; + mg_fmg_prev_rms = -1.0; + mg_fmg_stall_count = 0; + mg_fmg_promoted_on_stall = false; } /*--- Use the level-0 flow CFL as the base reference and derive the steady-state @@ -366,6 +372,30 @@ void CMultiGridIntegration::MultiGrid_Iteration(CGeometry ****geometry, } else if (conv_now <= 1e-2 * mg_conv_field_start_rms) { mg_conv_field_early_exit = true; } + + /*--- Stagnation promotion. The two criteria above are a fixed iteration budget and a fixed + * residual drop, and neither scales with the mesh: MG_Startup_Iter that is well matched on + * a medium grid leaves a fine grid grinding through a warmup that stopped paying off long + * before the budget ran out. What actually matters in FMG is when the coarse level stops + * reducing the error usefully - past that point only the finer level can make progress, so + * promote as soon as the per-iteration reduction stalls. + * + * A single slow iteration is not stagnation, so a run of them is required. Both the ratio + * and the run length are configurable, and MG_Startup_Iter still caps the window. ---*/ + const auto& mgOptsFMG = config[iZone]->GetMGOptions(); + const passivedouble stall_tol = SU2_TYPE::GetValue(mgOptsFMG.MG_Startup_Stagnation); + if (stall_tol > 0.0 && mg_fmg_prev_rms > 0.0) { + if (conv_now >= stall_tol * mg_fmg_prev_rms) + mg_fmg_stall_count++; + else + mg_fmg_stall_count = 0; + + if (mg_fmg_stall_count >= mgOptsFMG.MG_Startup_Stagnation_Iter) { + mg_conv_field_early_exit = true; + mg_fmg_promoted_on_stall = true; + } + } + mg_fmg_prev_rms = conv_now; } END_SU2_OMP_SAFE_GLOBAL_ACCESS } @@ -986,18 +1016,48 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); + /*--- Optional cap on how much one coarse-grid correction may move the solution at a point. + * Without it the only guard below is the NaN check, and a correction can drive a cell + * non-physical in a single application - measured on the turbulent flat plate, a W-cycle + * correction cuts wall-adjacent density by 25%, from which the energy equation never + * recovers. The cap is relative and per point, and the whole correction vector is scaled by + * one factor so its direction is preserved (scaling components independently would rotate + * the correction and break the coupling between the equations). + * + * Components that are negligible against the largest one at that point (transverse momentum + * in a freestream cell, say) carry no meaningful relative bound and are skipped, otherwise + * they would veto every correction. ---*/ + const su2double limit = config->GetMGOptions().MG_Correction_Limit; + const bool limiting = (limit > 0.0); + SU2_OMP_FOR_STAT(roundUpDiv(geo_fine->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Fine = 0ul; Point_Fine < geo_fine->GetnPointDomain(); Point_Fine++) { auto* Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); auto* Solution_Fine = sol_fine->GetNodes()->GetSolution(Point_Fine); + + /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ for (auto iVar = 0u; iVar < nVar; iVar++) { - /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ if (Residual_Fine[iVar] != Residual_Fine[iVar]) Residual_Fine[iVar] = 0.0; + } - su2double correction = factor * Residual_Fine[iVar]; - Solution_Fine[iVar] += correction; + su2double omega = 1.0; + if (limiting) { + su2double ref = 0.0; + for (auto iVar = 0u; iVar < nVar; iVar++) + ref = max(ref, fabs(Solution_Fine[iVar])); + + for (auto iVar = 0u; iVar < nVar; iVar++) { + const su2double scale = fabs(Solution_Fine[iVar]); + if (scale < 1e-6 * ref) continue; + const su2double correction = fabs(factor * Residual_Fine[iVar]); + if (correction > limit * scale) + omega = min(omega, limit * scale / correction); + } } + + for (auto iVar = 0u; iVar < nVar; iVar++) + Solution_Fine[iVar] += omega * factor * Residual_Fine[iVar]; } END_SU2_OMP_FOR From ae01006973830a2057a6e131d0a0c2d44986eb9a Mon Sep 17 00:00:00 2001 From: bigfooted Date: Tue, 18 Aug 2026 22:18:07 +0200 Subject: [PATCH 30/54] fix implicit lines in 3D for a large part. --- Common/include/option_structure.hpp | 5 +- Common/src/CConfig.cpp | 6 +- Common/src/geometry/CMultiGridGeometry.cpp | 491 ++++++++++++--------- SU2_CFD/src/solvers/CSolver.cpp | 11 +- 4 files changed, 301 insertions(+), 212 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index f3af3d16446..e2685d574ba 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1116,7 +1116,7 @@ inline SST_ParsedOptions ParseSSTOptions(const SST_OPTIONS *SST_Options, unsigne struct CMGOptions { su2double MG_Smooth_Res_Threshold{0.0}; /*!< \brief RMS reduction threshold for MG smoothing early exit. */ su2double MG_Smooth_Coeff{0.0}; /*!< \brief Jacobi smoother coefficient for coarse-grid correction. */ - unsigned long MG_Min_MeshSize{0}; /*!< \brief Minimum CVs on coarsest MG level. */ + unsigned long MG_Min_MeshSize{0}; /*!< \brief Minimum CVs on coarsest MG level, per MPI rank. */ std::vector MG_PreSmooth; /*!< \brief Multigrid pre-smoothing iterations per level. */ std::vector MG_PostSmooth; /*!< \brief Multigrid post-smoothing iterations per level. */ std::vector MG_CorrecSmooth; /*!< \brief Multigrid Jacobi correction-smoothing per level. */ @@ -1127,6 +1127,9 @@ struct CMGOptions { bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ + unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse + CV tangential to the wall. 0 = dimension-appropriate default + (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index e0b53ff79ba..e1ec0075034 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2076,7 +2076,7 @@ void CConfig::SetConfig_Options() { * prolongated multigrid correction, e.g. 0.1 caps it at 10%. The whole correction vector at a point is scaled by one * factor so its direction is preserved. 0 disables the limiter (previous behaviour). DEFAULT: 0 \ingroup Config*/ addDoubleOption("MG_CORRECTION_LIMIT", MGOptions.MG_Correction_Limit, 0.0); - /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level. Levels that would produce fewer CVs are not created. DEFAULT: 50 \ingroup Config*/ + /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); @@ -2084,6 +2084,10 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 20); /*!\brief MG_IMPLICIT_LINES_ISOTROPIC\n DESCRIPTION: Use isotropic agglomeration along implicit lines (4 cells per coarse CV) instead of anisotropic (2 cells per coarse CV). DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); + /*!\brief MG_IMPLICIT_LINES_MAX_GROUP\n DESCRIPTION: Maximum number of parallel implicit lines merged tangential to + * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). + * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ + addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_GROUP", MGOptions.MG_Implicit_Lines_Max_Group, 0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); /*!\brief MG_STARTUP_STAGNATION\n DESCRIPTION: Full-MG promotion on stagnation. If the active level's residual ratio diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index b97d919385b..5fff41d875b 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -88,6 +88,16 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- STEP 0: agglomerate the stretched layer above viscous walls along implicit lines, wall CV + * included. This runs before the general boundary agglomeration below so that the wall control + * volume and the layers stacked on top of it share one footprint; letting the general scheme + * claim the wall first would fix a footprint chosen without any knowledge of the lines, and the + * stack above it could then only be misaligned with its own base. Everything it claims is + * already marked agglomerated, so the boundary and interior passes below simply skip it. ---*/ + if (config->GetMGOptions().MG_Implicit_Lines) { + AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, MGQueue_InnerCV); + } + /*--- STEP 1: The first step is the boundary agglomeration. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { /*--- Skip periodic boundaries: do not agglomerate on periodic markers. ---*/ @@ -171,15 +181,22 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Note that in 2D, this is a corner and we do not agglomerate unless one of them is SEND_RECEIVE. ---*/ - /*--- In 3D, we agglomerate if the 2 markers are the same. ---*/ + /*--- Note that in 2D, this is a corner and we do not agglomerate unless one of the two markers + is SEND_RECEIVE, i.e. the point lies on a physical boundary that happens to be split by + an MPI partition interface. ---*/ if (counter == 2) { if (nDim == 2) { agglomerate_seed = ((config->GetMarker_All_KindBC(copy_marker[0]) == SEND_RECEIVE) || (config->GetMarker_All_KindBC(copy_marker[1]) == SEND_RECEIVE)); } - /*--- agglomerate if both markers are the same. ---*/ - if (nDim == 3) agglomerate_seed = (copy_marker[0] == copy_marker[1]); + /*--- In 3D, this is a ridge point (an edge feature where two surface markers meet). + Always allow it to attempt agglomeration here; SetBoundAgglomeration() enforces + the actual ridge-ridge rule downstream: it may only pair with a neighboring ridge + point that carries the identical marker pair (or the same physical marker plus a + SEND_RECEIVE halo marker). A mismatched marker pair usually indicates a genuine + sharp corner in the geometry and is correctly left un-merged (falls through to the + singleton leftover loop). ---*/ + if (nDim == 3) agglomerate_seed = true; /*--- Euler walls: check curvature-based agglomeration criterion for both markers ---*/ // only in 3d because in 2d it's a corner @@ -311,11 +328,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Agglomerate high-aspect-ratio interior nodes along implicit lines from walls. ---*/ - if (config->GetMGOptions().MG_Implicit_Lines) { - AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, MGQueue_InnerCV); - } - /*--- STEP 2: Agglomerate the domain points. ---*/ auto iteration = 0ul; @@ -651,10 +663,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- Console output with the summary of the agglomeration ---*/ unsigned long nPointFine = fine_grid->GetnPointDomain(); - unsigned long Global_nPointCoarse, Global_nPointFine; + unsigned long Global_nPointCoarse, Global_nPointFine, Min_nPointCoarse; SU2_MPI::Allreduce(&nPointDomain, &Global_nPointCoarse, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&nPointFine, &Global_nPointFine, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + SU2_MPI::Allreduce(&nPointDomain, &Min_nPointCoarse, 1, MPI_UNSIGNED_LONG, MPI_MIN, SU2_MPI::GetComm()); SetGlobal_nPointDomain(Global_nPointCoarse); @@ -666,11 +679,15 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un const su2double ratio = su2double(Global_nPointFine) / su2double(Global_nPointCoarse); - if (Global_nPointCoarse < config->GetMGOptions().MG_Min_MeshSize) { + /*--- Stop coarsening once the smallest per-rank partition falls below the minimum, + not just the summed total, since each rank runs its own MG hierarchy locally + and a partition that agglomerates down to too few CVs degenerates the operator + on that rank even if other ranks still have plenty of points. ---*/ + if (Min_nPointCoarse < config->GetMGOptions().MG_Min_MeshSize) { if (rank == MASTER_NODE) - cout << "MG level " << iMesh << " has only " << Global_nPointCoarse - << " CVs (< MG_MIN_MESHSIZE=" << config->GetMGOptions().MG_Min_MeshSize << "). Reducing MG levels to " - << iMesh - 1 << "." << endl; + cout << "MG level " << iMesh << " has only " << Min_nPointCoarse + << " CVs on the smallest partition (< MG_MIN_MESHSIZE=" << config->GetMGOptions().MG_Min_MeshSize + << "). Reducing MG levels to " << iMesh - 1 << "." << endl; config->SetMGLevels(iMesh - 1); } else if (rank == MASTER_NODE) { PrintingToolbox::CTablePrinter MGTable(&std::cout); @@ -1281,253 +1298,309 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, CMultiGridQueue& MGQueue_InnerCV) { /*--- Parameters ---*/ - const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop line if direction deviates more than this. */ + const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop a line if the direction deviates more than this. */ const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; const unsigned long nPointFine = fine_grid->GetnPoint(); - const unsigned long starting_Index_CoarseCV = Index_CoarseCV; /*--- Track how many CVs we create ---*/ - const bool DEBUG_OUTPUT = (rank == MASTER_NODE); /*--- Enable detailed diagnostic output ---*/ - - /*--- Collect implicit lines starting at viscous (no-slip) wall vertices only. - * Seeding from non-wall boundaries (farfield, inlet, outlet, symmetry) would - * claim interior BL cells before the wall lines can reach them, leaving - * wall-seeded lines with length < 3 (discarded). Restricting to viscous walls - * ensures the boundary-layer cells are agglomerated wall-first. - * Each line: [wall_node, interior_1, interior_2, ...]. - * The wall node (index 0) is already agglomerated by boundary agglomeration; - * only interior nodes (index >= 1) are paired into coarse CVs. ---*/ - vector> lines; + const unsigned long starting_Index_CoarseCV = Index_CoarseCV; + const bool DEBUG_OUTPUT = (rank == MASTER_NODE); + + /*--- How many parallel lines one coarse CV may span tangential to the wall. In 2D a wall "face" is + * an edge with 2 end nodes, in 3D a quadrilateral with 4 corner nodes, which is the number of + * lines that must be bundled to coarsen by 2 in every wall-tangential direction. ---*/ + unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; + if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; + + /*================================================================================================== + * PHASE A - build the implicit lines. + * + * Lines are grown one step at a time across ALL lines simultaneously rather than one line to + * completion at a time, and a node is claimed globally the moment any line takes it. Growing them + * one-at-a-time lets an early line run the full depth of the layer and consume nodes that a later, + * neighbouring line needed, so that later line terminates after a step or two; the lines then have + * wildly different lengths and cannot be bundled into columns of uniform depth. Advancing in + * lockstep makes all lines compete for each layer on equal terms, which on an extruded prismatic + * layer reproduces the mesh's own structure: every line reaches the same depth. + *================================================================================================*/ + vector> lines; /*!< lines[i] = [wall_node, interior_1, interior_2, ...] */ + vector dir; /*!< Current marching direction of each line, nDim per line. */ + vector claimed(nPointFine, 0); /*!< Node already belongs to some line. */ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - /*--- Only seed lines from viscous (no-slip) wall markers. - * Non-wall boundaries (farfield, inlet, outlet, symmetry) must NOT seed - * lines because they would prematurely claim boundary-layer interior nodes. ---*/ + /*--- Seed only from viscous (no-slip) walls: those are the boundaries with a stretched layer + * above them. Seeding from farfield/inlet/outlet/symmetry would claim layer nodes before the + * wall lines could reach them. ---*/ const auto bc = config->GetMarker_All_KindBC(iMarker); if (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; + if (claimed[iPoint]) continue; /*--- A node on two wall markers must seed only one line. ---*/ - /*--- Get vertex normal to seed the line direction ---*/ const long ChildVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); if (ChildVertex == -1) continue; su2double Normal[MAXNDIM] = {0.0}; fine_grid->vertex[iMarker][ChildVertex]->GetNormal(Normal); + const su2double nrm = GeometryToolbox::Norm(nDim, Normal); + if (nrm <= 0.0) continue; - /*--- Normalize the direction ---*/ - su2double prev_dir[MAXNDIM] = {0.0}; - su2double norm_prev = 0.0; - for (unsigned short d = 0; d < nDim; ++d) { - prev_dir[d] = Normal[d]; - norm_prev += Normal[d] * Normal[d]; - } - if (norm_prev <= 0.0) continue; - norm_prev = sqrt(norm_prev); - for (unsigned short d = 0; d < nDim; ++d) prev_dir[d] /= norm_prev; - - /*--- Build the implicit line by following the best-aligned interior neighbor. ---*/ - vector L; - L.push_back(iPoint); - auto current = iPoint; - - while (L.size() < MAX_LINE_LENGTH) { - su2double best_dot = -2.0; - unsigned long best_neighbor = ULONG_MAX; - - for (auto jPoint : fine_grid->nodes->GetPoints(current)) { - if (!fine_grid->nodes->GetDomain(jPoint)) continue; - if (fine_grid->nodes->GetBoundary(jPoint)) continue; - if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; - if (find(L.begin(), L.end(), jPoint) != L.end()) continue; - - /*--- Compute normalized direction to candidate ---*/ - su2double vec[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(current), vec); - const su2double len = GeometryToolbox::Norm(nDim, vec); - if (len <= 0.0) continue; - for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - - /*--- Alignment with previous direction ---*/ - const su2double dot = GeometryToolbox::DotProduct(nDim, vec, prev_dir); - if (dot > best_dot) { - best_dot = dot; - best_neighbor = jPoint; - } - } - - if (best_neighbor == ULONG_MAX || best_dot < cos_threshold) break; + lines.push_back({iPoint}); + claimed[iPoint] = 1; + for (unsigned short d = 0; d < nDim; ++d) dir.push_back(Normal[d] / nrm); + } + } - L.push_back(best_neighbor); + if (lines.empty()) return; - /*--- Update direction for next step ---*/ - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(best_neighbor), fine_grid->nodes->GetCoord(current), - prev_dir); - const su2double len = GeometryToolbox::Norm(nDim, prev_dir); - if (len <= 0.0) break; - for (unsigned short d = 0; d < nDim; ++d) prev_dir[d] /= len; + vector growing(lines.size(), 1); + for (bool any_grew = true; any_grew;) { + any_grew = false; + for (unsigned long li = 0; li < lines.size(); ++li) { + if (!growing[li]) continue; + if (lines[li].size() >= MAX_LINE_LENGTH) { + growing[li] = 0; + continue; + } + const auto current = lines[li].back(); + su2double best_dot = -2.0; + unsigned long best_neighbor = ULONG_MAX; + + for (auto jPoint : fine_grid->nodes->GetPoints(current)) { + if (!fine_grid->nodes->GetDomain(jPoint)) continue; + if (fine_grid->nodes->GetBoundary(jPoint)) continue; + if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; + if (claimed[jPoint]) continue; + + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(current), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) continue; + for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; + + const su2double dot = GeometryToolbox::DotProduct(nDim, vec, &dir[li * nDim]); + if (dot > best_dot) { + best_dot = dot; + best_neighbor = jPoint; + } + } - current = best_neighbor; + if (best_neighbor == ULONG_MAX || best_dot < cos_threshold) { + growing[li] = 0; + continue; } - /*--- Accept only lines with at least 2 interior nodes (length >= 3 including wall) ---*/ - if (L.size() >= 3) { - lines.push_back(std::move(L)); + su2double step[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(best_neighbor), fine_grid->nodes->GetCoord(current), + step); + const su2double slen = GeometryToolbox::Norm(nDim, step); + if (slen <= 0.0) { + growing[li] = 0; + continue; } + for (unsigned short d = 0; d < nDim; ++d) dir[li * nDim + d] = step[d] / slen; + + lines[li].push_back(best_neighbor); + claimed[best_neighbor] = 1; + any_grew = true; } } + /*--- A line needs at least one interior node to contribute anything. Drop the rest and release + * their wall seed so ordinary boundary agglomeration can treat it normally. ---*/ + { + vector> kept; + kept.reserve(lines.size()); + for (auto& L : lines) { + if (L.size() >= 2) + kept.push_back(std::move(L)); + else + claimed[L[0]] = 0; + } + lines = std::move(kept); + } if (lines.empty()) return; - /*--- Agglomeration strategy: at each "position" (distance from the wall along a - * line) every line contributes a block of nBlock consecutive nodes; two lines - * are paired through a mesh-adjacency search of their block anchors and the - * combined blocks become the children of one coarse CV. + /*================================================================================================== + * PHASE B - partition the lines into bundles. * - * ANISOTROPIC (default, nBlock=1): pair single nodes at the SAME distance from - * the wall on two DIFFERENT lines. Each coarse CV has 2 fine children. - * Reduces the mesh by a factor ~2 normal to the wall, preserves resolution - * along the wall. + * Every line must end up in exactly one bundle, and a bundle must be a compact patch on the wall: + * in 3D the four lines rising from the corners of one wall quadrilateral, in 2D the two lines from + * the ends of one wall edge. Selecting, for each line independently, some set of neighbours to + * merge with does not do this - the relation is not symmetric, so line 1 claiming {2,3,4} does not + * stop line 2 from claiming {1,3,5}, and the bundles overlap and fight over nodes. * - * ISOTROPIC (nBlock=2): pair 2-node blocks (2 positions x 2 lines) into one - * coarse CV with 4 fine children. Reduces the mesh uniformly by a factor - * ~4 in all directions. ---*/ - const unsigned long nBlock = ISOTROPIC ? 2 : 1; - vector reserved(nPointFine, 0); - unsigned long position_idx = 0; - - /*--- Fine-grid nodes line `li` contributes at the current position, or empty if - the line is too short to reach that far. ---*/ - auto LineBlock = [&](unsigned long li) -> vector { - const auto& L = lines[li]; - const unsigned long first = 1 + nBlock * position_idx; - if (L.size() < first + nBlock) return {}; - return vector(L.begin() + first, L.begin() + first + nBlock); - }; - - while (true) { - /*--- Cache each line's block for this position and collect the active ones. ---*/ - vector> block(lines.size()); - vector active_lines; - active_lines.reserve(lines.size()); - for (unsigned long li = 0; li < lines.size(); ++li) { - block[li] = LineBlock(li); - if (!block[li].empty()) active_lines.push_back(li); + * Building the partition by repeated pairwise matching avoids that by construction. One matching + * round pairs adjacent lines into 2-bundles (the wall edge); a second round pairs adjacent + * 2-bundles into 4-bundles (the wall quadrilateral). Each round is a matching, so membership is + * mutually exclusive at every stage and the result is a true partition. It also needs nothing but + * point-to-point connectivity, so it works identically on every multigrid level - boundary face + * connectivity does not exist on agglomerated grids, so a literal "same quadrilateral" test would + * only ever work for the first coarsening. + * + * Lines may only be bundled when their wall nodes carry the same set of physical markers, so that + * a bundle never straddles a boundary-condition change (the same rule ordinary agglomeration uses: + * ridges merge only with ridges, valleys only with valleys). + *================================================================================================*/ + const unsigned long nLines = lines.size(); + + /*--- Marker signature of each line's wall node. ---*/ + vector> sig(nLines); + for (unsigned long li = 0; li < nLines; ++li) { + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + if (fine_grid->nodes->GetVertex(lines[li][0], iMarker) != -1) sig[li].push_back(iMarker); } - if (active_lines.empty()) break; + sort(sig[li].begin(), sig[li].end()); + } - vector line_processed(lines.size(), 0); - bool any_work = false; + /*--- Line adjacency, inherited from the wall nodes' mesh connectivity. ---*/ + unordered_map lineOfWallNode; + lineOfWallNode.reserve(nLines); + for (unsigned long li = 0; li < nLines; ++li) lineOfWallNode[lines[li][0]] = li; + + vector> adj(nLines); + for (unsigned long li = 0; li < nLines; ++li) { + for (auto jPoint : fine_grid->nodes->GetPoints(lines[li][0])) { + const auto it = lineOfWallNode.find(jPoint); + if (it != lineOfWallNode.end() && it->second != li) adj[li].push_back(it->second); + } + } - for (auto li1 : active_lines) { - if (line_processed[li1]) continue; + /*--- Round 1: match adjacent lines into pairs. ---*/ + vector> bundles; + vector bundleOf(nLines, -1); + bundles.reserve(nLines); + for (unsigned long li = 0; li < nLines; ++li) { + if (bundleOf[li] >= 0) continue; + const long b = static_cast(bundles.size()); + bundles.push_back({li}); + bundleOf[li] = b; + if (max_group < 2) continue; + for (auto lj : adj[li]) { + if (bundleOf[lj] >= 0 || sig[lj] != sig[li]) continue; + bundles[b].push_back(lj); + bundleOf[lj] = b; + break; + } + } - const auto anchor = block[li1].front(); - if (fine_grid->nodes->GetAgglomerate(anchor) || reserved[anchor]) continue; + /*--- Further rounds: merge adjacent bundles while they still fit. In 3D this turns pairs into + * quadrilaterals; in 2D max_group is 2 so it only absorbs leftover singletons. ---*/ + for (bool changed = true; changed;) { + changed = false; + + vector> badj(bundles.size()); + for (unsigned long li = 0; li < nLines; ++li) + for (auto lj : adj[li]) + if (bundleOf[li] != bundleOf[lj]) badj[bundleOf[li]].push_back(bundleOf[lj]); + for (auto& v : badj) { + sort(v.begin(), v.end()); + v.erase(unique(v.begin(), v.end()), v.end()); + } - /*--- Find an unprocessed neighboring line: one whose block anchor is a - mesh-neighbor of ours at the same position. ---*/ - unsigned long li2 = ULONG_MAX; - for (auto neighbor_point : fine_grid->nodes->GetPoints(anchor)) { - for (auto candidate : active_lines) { - if (candidate == li1 || line_processed[candidate]) continue; - if (block[candidate].front() == neighbor_point) { - li2 = candidate; - break; - } - } - if (li2 != ULONG_MAX) break; + vector consumed(bundles.size(), 0); + vector> merged; + merged.reserve(bundles.size()); + for (unsigned long b = 0; b < bundles.size(); ++b) { + if (consumed[b]) continue; + consumed[b] = 1; + auto group = bundles[b]; + for (auto h : badj[b]) { + if (consumed[h]) continue; + if (group.size() + bundles[h].size() > max_group) continue; + if (sig[bundles[h].front()] != sig[group.front()]) continue; + consumed[h] = 1; + group.insert(group.end(), bundles[h].begin(), bundles[h].end()); + changed = true; + break; } + merged.push_back(std::move(group)); + } - if (li2 == ULONG_MAX) { - if (DEBUG_OUTPUT && position_idx < 3) { - cout << " Line " << li1 << " at position " << position_idx << " (node " << anchor - << ") has NO neighbor line!" << endl; - } - continue; - } + bundles = std::move(merged); + for (unsigned long b = 0; b < bundles.size(); ++b) + for (auto li : bundles[b]) bundleOf[li] = static_cast(b); + } - /*--- Assemble and validate the coarse CV's children together. ---*/ - auto group = block[li1]; - group.insert(group.end(), block[li2].begin(), block[li2].end()); + /*================================================================================================== + * PHASE C - extrude each bundle into a stack of coarse control volumes. + * + * The bundle's wall nodes become one coarse CV, and each successive layer of the bundle's lines + * becomes the next, so the coarse grid inherits the layer structure of the fine grid and a line + * relaxation remains meaningful on it. Because Phase A made the lines node-disjoint and Phase B + * made the bundles a partition, no two bundles can ever contend for the same node, so a stack is + * never interrupted part way up. + * + * The multigrid queue is deliberately not updated here: the sync loop that follows the boundary + * agglomeration removes every point already marked agglomerated, so removing them a second time + * from this function would be an error. + *================================================================================================*/ + const unsigned long nBlock = ISOTROPIC ? 2 : 1; - bool valid = true; - for (auto p : group) { - if (fine_grid->nodes->GetAgglomerate(p) || reserved[p] || !GeometricalCheck(p, fine_grid, config)) { - valid = false; - break; - } - } - if (!valid) continue; // one of the nodes wasn't ready; li2 may still pair elsewhere. + map bundle_size_histogram; + unsigned long nStacks = 0, nTruncated = 0; - /*--- Guard against the same fine point appearing in both blocks - (can happen if two lines' walks overlap in space). ---*/ - bool duplicate = false; - for (size_t i = 0; !duplicate && i + 1 < group.size(); ++i) - for (size_t j = i + 1; j < group.size(); ++j) - if (group[i] == group[j]) duplicate = true; + for (const auto& members : bundles) { + bundle_size_histogram[members.size()]++; - if (duplicate) { - line_processed[li1] = line_processed[li2] = 1; - continue; - } + /*--- The wall CV. Claiming it here, before ordinary boundary agglomeration runs, is what keeps + * the whole stack aligned: the layer above a wall CV has exactly the same footprint. ---*/ + bool valid = true; + for (auto li : members) + if (!GeometricalCheck(lines[li][0], fine_grid, config)) valid = false; + if (!valid) continue; - /*--- Create the coarse CV from the combined blocks. ---*/ - for (size_t c = 0; c < group.size(); ++c) { + for (unsigned long c = 0; c < members.size(); ++c) { + const auto p = lines[members[c]][0]; + fine_grid->nodes->SetParent_CV(p, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, c, p); + } + nodes->SetnChildren_CV(Index_CoarseCV, static_cast(members.size())); + Index_CoarseCV++; + nStacks++; + + /*--- The interior layers, in lockstep, to the end of the shortest line in the bundle. ---*/ + unsigned long shortest = ULONG_MAX; + for (auto li : members) shortest = std::min(shortest, lines[li].size()); + + unsigned long placed = 1; + for (unsigned long first = 1; first + nBlock <= shortest; first += nBlock) { + vector group; + group.reserve(members.size() * nBlock); + for (auto li : members) + for (unsigned long b = 0; b < nBlock; ++b) group.push_back(lines[li][first + b]); + + valid = true; + for (auto p : group) + if (!GeometricalCheck(p, fine_grid, config)) valid = false; + if (!valid) break; + + for (unsigned long c = 0; c < group.size(); ++c) { fine_grid->nodes->SetParent_CV(group[c], Index_CoarseCV); nodes->SetChildren_CV(Index_CoarseCV, c, group[c]); - reserved[group[c]] = 1; - MGQueue_InnerCV.RemoveCV(group[c]); } nodes->SetnChildren_CV(Index_CoarseCV, static_cast(group.size())); Index_CoarseCV++; - - line_processed[li1] = line_processed[li2] = 1; - any_work = true; + placed = first + nBlock; } - if (!any_work) break; - position_idx++; - } - - /*--- Count how many CVs and nodes were created ---*/ - const auto nCVs_created = Index_CoarseCV - starting_Index_CoarseCV; - unsigned long nNodes_claimed = 0; - unsigned long nNodes_on_lines = 0; - unsigned long nNodes_unpaired = 0; - - for (const auto& L : lines) { - for (size_t i = 1; i < L.size(); ++i) { // Skip wall node at [0] - nNodes_on_lines++; - if (!reserved[L[i]]) nNodes_unpaired++; - } - } - - for (unsigned long i = 0; i < nPointFine; ++i) { - if (reserved[i]) nNodes_claimed++; + /*--- Nodes above the shortest line, or above a block that did not divide evenly, are left to + * ordinary domain agglomeration. ---*/ + for (auto li : members) nTruncated += lines[li].size() - placed; } if (DEBUG_OUTPUT) { - cout << " Created " << nCVs_created << " coarse CVs from " << nNodes_claimed << " fine nodes." << endl; - cout << " Nodes on implicit lines: " << nNodes_on_lines << " (paired=" << (nNodes_on_lines - nNodes_unpaired) - << ", unpaired=" << nNodes_unpaired << ")" << endl; - - if (nNodes_unpaired > 0) { - cout << " WARNING: " << nNodes_unpaired << " nodes on implicit lines were left unpaired!" << endl; - cout << " These will be processed by domain agglomeration (may create wrong orientation)." << endl; - } - } - - /*--- Verify all claimed nodes are properly marked as agglomerated (SetParent_CV should - guarantee this; a mismatch would indicate a bookkeeping bug above). ---*/ - unsigned long mismatches = 0; - for (unsigned long i = 0; i < nPointFine; ++i) { - if (reserved[i] && !fine_grid->nodes->GetAgglomerate(i)) { - mismatches++; - } - } - if (mismatches > 0 && DEBUG_OUTPUT) { - cout << " WARNING: " << mismatches << " nodes marked as reserved but not agglomerated!" << endl; + unsigned long nLineNodes = 0; + for (const auto& L : lines) nLineNodes += L.size(); + cout << " Implicit lines: " << nLines << " lines, " << nStacks << " stacks, bundle sizes "; + for (const auto& h : bundle_size_histogram) cout << h.first << "x" << h.second << " "; + cout << "\n Coarse CVs from lines: " << (Index_CoarseCV - starting_Index_CoarseCV) << " covering " + << (nLineNodes - nTruncated) << "/" << nLineNodes << " line nodes"; + if (nTruncated > 0) cout << " (" << nTruncated << " left to domain agglomeration)"; + cout << endl; } } diff --git a/SU2_CFD/src/solvers/CSolver.cpp b/SU2_CFD/src/solvers/CSolver.cpp index e014c794a52..b16f1df825e 100644 --- a/SU2_CFD/src/solvers/CSolver.cpp +++ b/SU2_CFD/src/solvers/CSolver.cpp @@ -1990,7 +1990,16 @@ void CSolver::AdaptCFLNumber(CGeometry **geometry, void CSolver::SetResidual_RMS(const CGeometry *geometry, const CConfig *config) { SU2_ZONE_SCOPED - if (geometry->GetMGLevel() != MESH_0 && !config->GetMGOptions().MG_Smooth_EarlyExit) return; + /*--- On coarse levels the reduction is normally skipped for performance, unless + * MG_Smooth_EarlyExit needs it, or a Full-MG cycle needs a globally consistent + * residual to decide level promotion (CMultiGridIntegration::SetFullMultigrid_Solver). + * Skipping it there would leave Residual_RMS as each rank's local, un-reduced + * accumulator, letting ranks disagree on when to promote and desynchronizing the + * point-to-point communication pattern between ranks. ---*/ + const bool fmg_needs_reduction = geometry->GetMGLevel() != MESH_0 && + config->GetMGCycle() == MG_CYCLE::FULL && + config->GetFinestMesh() == geometry->GetMGLevel(); + if (geometry->GetMGLevel() != MESH_0 && !config->GetMGOptions().MG_Smooth_EarlyExit && !fmg_needs_reduction) return; BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { From 0529c94290bdb224c5284b2628f97bc010395edc Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 19 Aug 2026 19:40:00 +0200 Subject: [PATCH 31/54] fix mg+mpi interface issue --- Common/src/geometry/CMultiGridGeometry.cpp | 139 +++++++++++++++------ 1 file changed, 98 insertions(+), 41 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 5fff41d875b..5d967415b71 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -103,6 +103,13 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- Skip periodic boundaries: do not agglomerate on periodic markers. ---*/ if (config->GetMarker_All_KindBC(iMarker) == PERIODIC_BOUNDARY) continue; + /*--- Skip SEND_RECEIVE markers. Carrying one does not put a point on a boundary, it only + * records that the point is mirrored on another rank. A point whose only markers are + * SEND_RECEIVE is an interior point, and is left to the domain pass (STEP 2) which is + * where a serial run would agglomerate it too. Points that do sit on a physical boundary + * are still reached here through their physical marker. ---*/ + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); @@ -128,13 +135,18 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned short copy_marker[3] = {}; marker_seed.push_back(iMarker); - /*--- For a particular point in the fine grid we save all the markers - that are in that point ---*/ + /*--- For a particular point in the fine grid we save all the physical markers that are in + that point. SEND_RECEIVE markers are deliberately not counted: including them would make + an ordinary wall point look like a ridge, and a wall/symmetry ridge look like a corner + (which the counter > 2 rule below then refuses to agglomerate at all), so a point would be + classified differently depending only on where the partition happens to cut. ---*/ for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker(); jMarker++) { - const string Marker_Tag = config->GetMarker_All_TagBound(iMarker); + if (config->GetMarker_All_KindBC(jMarker) == SEND_RECEIVE) continue; if (fine_grid->nodes->GetVertex(iPoint, jMarker) != -1) { - copy_marker[counter] = jMarker; + /*--- Count every physical marker, the counter > 2 test needs the true count, but only + store the first few, which is all the matching rules ever look at. ---*/ + if (counter < 3) copy_marker[counter] = jMarker; counter++; if (jMarker != iMarker) { @@ -173,29 +185,20 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un euler_wall_agglomerated[marker_seed[0]]++; } } - - /*--- Note that if the (single) marker is a SEND_RECEIVE, then the node is actually an interior point. - In that case it can only be agglomerated with another interior point. ---*/ - if (config->GetMarker_All_KindBC(marker_seed[0]) == SEND_RECEIVE) { - agglomerate_seed = true; - } } - /*--- Note that in 2D, this is a corner and we do not agglomerate unless one of the two markers - is SEND_RECEIVE, i.e. the point lies on a physical boundary that happens to be split by - an MPI partition interface. ---*/ + /*--- Two physical markers meet here. ---*/ if (counter == 2) { - if (nDim == 2) { - agglomerate_seed = ((config->GetMarker_All_KindBC(copy_marker[0]) == SEND_RECEIVE) || - (config->GetMarker_All_KindBC(copy_marker[1]) == SEND_RECEIVE)); - } + /*--- In 2D that is a genuine corner in the geometry, which is never agglomerated. A wall + point merely split by a partition interface no longer reaches this branch: it counts + one physical marker and is handled above as the valley point it is. ---*/ + if (nDim == 2) agglomerate_seed = false; /*--- In 3D, this is a ridge point (an edge feature where two surface markers meet). Always allow it to attempt agglomeration here; SetBoundAgglomeration() enforces the actual ridge-ridge rule downstream: it may only pair with a neighboring ridge - point that carries the identical marker pair (or the same physical marker plus a - SEND_RECEIVE halo marker). A mismatched marker pair usually indicates a genuine - sharp corner in the geometry and is correctly left un-merged (falls through to the - singleton leftover loop). ---*/ + point that carries the identical physical marker pair. A mismatched marker pair + usually indicates a genuine sharp corner in the geometry and is correctly left + un-merged (falls through to the singleton leftover loop). ---*/ if (nDim == 3) agglomerate_seed = true; /*--- Euler walls: check curvature-based agglomeration criterion for both markers ---*/ @@ -298,6 +301,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un i.e. make one coarse CV with a single child. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + /*--- As in STEP 1, a SEND_RECEIVE marker does not make a point a boundary point. Turning the + leftovers of those markers into single-child coarse CVs here would strand every interior point + along a partition interface before the domain pass below ever gets to see it. ---*/ + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); @@ -445,8 +453,30 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un SetPoint_Connectivity(fine_grid); + /*--- The connectivity just built only knows about coarse CVs of this rank: the halo CVs do not + exist until the MPI relay below runs, and the relay cannot run earlier because it broadcasts the + parent indices that the merge here is still free to change. So a CV touching a partition boundary + may look isolated while actually having neighbors on the other rank, and merging it would be + wrong. Mark those CVs and leave them alone; a genuinely isolated CV in the interior is unaffected. + Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, + but only for the CVs that really do border another rank rather than for every CV near one. ---*/ + + vector touchesPartition(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { - if (nodes->GetnPoint(iCoarsePoint) == 1) { + for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { + const auto iFinePoint = nodes->GetChildren_CV(iCoarsePoint, iChildren); + for (auto iFinePoint_Neighbor : fine_grid->nodes->GetPoints(iFinePoint)) { + if (fine_grid->nodes->GetParent_CV(iFinePoint_Neighbor) == std::numeric_limits::max()) { + touchesPartition[iCoarsePoint] = true; + break; + } + } + if (touchesPartition[iCoarsePoint]) break; + } + } + + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { + if ((nodes->GetnPoint(iCoarsePoint) == 1) && !touchesPartition[iCoarsePoint]) { /*--- Find the neighbor of the isolated point. This neighbor is the right control volume ---*/ const auto iCoarsePoint_Complete = nodes->GetPoint(iCoarsePoint, 0); @@ -781,18 +811,22 @@ bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vectornodes->GetBoundary(CVPoint)) { - /*--- Identify the markers of the vertex that we want to agglomerate ---*/ - - // count number of markers on the agglomeration candidate - for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker() && counter < 3; jMarker++) { + /*--- Identify the physical markers of the vertex that we want to agglomerate. SEND_RECEIVE + markers are skipped for the same reason as on the seed side: they say nothing about the + boundary condition the candidate carries, only that it is mirrored on another rank. A + candidate whose markers are all SEND_RECEIVE therefore ends up with counter == 0 and is + rejected below, which is the answer a serial run gives for the interior point it really is. ---*/ + + for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker(); jMarker++) { + if (config->GetMarker_All_KindBC(jMarker) == SEND_RECEIVE) continue; if (fine_grid->nodes->GetVertex(CVPoint, jMarker) != -1) { - copy_marker[counter] = jMarker; + if (counter < 3) copy_marker[counter] = jMarker; counter++; } } - /*--- The basic condition is that the agglomerated vertex must have the same physical marker, - but eventually a send-receive condition ---*/ + /*--- The basic condition is that the agglomerated vertex must have the same physical marker + as the seed. ---*/ /*--- Only one marker in the vertex that is going to be agglomerated ---*/ @@ -801,17 +835,11 @@ bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vectorGetMarker_All_KindBC(other_marker) == SEND_RECEIVE) { - agglomerate_CV = true; - } - } } /*--- If there are two markers in the vertex that is going to be aglomerated ---*/ @@ -821,7 +849,7 @@ bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vectornodes->GetPoints(iFinePoint)) { const auto iParent = fine_grid->nodes->GetParent_CV(iFinePoint_Neighbor); + /*--- Skip neighbors whose parent is not known yet. The first call to this function happens + during construction, before the MPI relay has assigned parents to the fine grid's halo + points, so those still hold the sentinel. Letting it through would add one fake neighbor to + every coarse CV along a partition boundary, which both corrupts nNeighbor and hides the CV + from the isolated-CV repair. The driver calls this again once the relay has run. ---*/ + if (iParent == std::numeric_limits::max()) continue; /*--- If it is not the target coarse point, it is a coarse neighbor. ---*/ if (iParent != iCoarsePoint) { /*--- Avoid duplicates. ---*/ @@ -1328,6 +1362,25 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, vector dir; /*!< Current marching direction of each line, nDim per line. */ vector claimed(nPointFine, 0); /*!< Node already belongs to some line. */ + /*--- Nodes that sit on a boundary with a boundary condition on it. A line must not grow into one, + * because those nodes belong to the boundary agglomeration and a stack that absorbed one would + * straddle two boundaries. CPoint's Boundary flag cannot answer this on its own: it is set for + * every marker a node belongs to, SEND_RECEIVE included, so on a partitioned mesh it is also + * true for the ordinary interior nodes of the send fringe. Testing it directly would stop every + * line that reaches the fringe one layer short of the partition, leaving the top of those + * columns to isotropic agglomeration purely because of where the mesh was cut. ---*/ + vector onPhysicalBoundary(nPointFine, 0); + for (auto iPoint = 0ul; iPoint < nPointFine; ++iPoint) { + if (!fine_grid->nodes->GetBoundary(iPoint)) continue; + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + if (fine_grid->nodes->GetVertex(iPoint, iMarker) != -1) { + onPhysicalBoundary[iPoint] = 1; + break; + } + } + } + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { /*--- Seed only from viscous (no-slip) walls: those are the boundaries with a stretched layer * above them. Seeding from farfield/inlet/outlet/symmetry would claim layer nodes before the @@ -1370,8 +1423,12 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long best_neighbor = ULONG_MAX; for (auto jPoint : fine_grid->nodes->GetPoints(current)) { + /*--- Halo nodes stay out: their parent is dictated by the rank that owns them and arrives + * through the MPI relay, so a line claiming one here would fight that assignment. A line + * therefore still ends at the partition itself, but now only there, instead of one layer + * earlier at the fringe of owned nodes. ---*/ if (!fine_grid->nodes->GetDomain(jPoint)) continue; - if (fine_grid->nodes->GetBoundary(jPoint)) continue; + if (onPhysicalBoundary[jPoint]) continue; if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; if (claimed[jPoint]) continue; From a029d61a653b85d07c3adb7fdabd4597cc821a71 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 19 Aug 2026 19:59:47 +0200 Subject: [PATCH 32/54] pre-commit --- Common/include/linear_algebra/CSysSolve.hpp | 4 +- Common/src/geometry/CGeometry.cpp | 9 ++- Common/src/geometry/CMultiGridGeometry.cpp | 79 ++++++++++++++++----- 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/Common/include/linear_algebra/CSysSolve.hpp b/Common/include/linear_algebra/CSysSolve.hpp index 337701c87b5..4191b0cf11c 100644 --- a/Common/include/linear_algebra/CSysSolve.hpp +++ b/Common/include/linear_algebra/CSysSolve.hpp @@ -140,8 +140,8 @@ class CSysSolve { * CSysMatrix (not in the short-lived CPreconditioner object built in Solve), so simply * skipping Build() reuses the previous one. This instance belongs to one solver on one * grid level, so the counter is naturally per-level. See MG_COARSE_PREC_FREEZE. ---*/ - mutable unsigned long precSolveCount = 0; /*!< \brief Linear solves done by this instance. */ - mutable bool buildPrecThisSolve = true; /*!< \brief Decision for the current solve, shared by all threads. */ + mutable unsigned long precSolveCount = 0; /*!< \brief Linear solves done by this instance. */ + mutable bool buildPrecThisSolve = true; /*!< \brief Decision for the current solve, shared by all threads. */ /*! * \brief sign transfer function diff --git a/Common/src/geometry/CGeometry.cpp b/Common/src/geometry/CGeometry.cpp index a64e6c5db91..6a62676ac9f 100644 --- a/Common/src/geometry/CGeometry.cpp +++ b/Common/src/geometry/CGeometry.cpp @@ -4505,11 +4505,10 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) const auto pct = [&](unsigned long n) { return 100.0 * passivedouble(n) / globalNLineLets; }; std::cout << "Computed linelet structure on MG level " << MGLevel << ", " << static_cast(passivedouble(globalNPoints) / globalNLineLets) - << " points in each line (average), " << globalMaxNPoints << " longest, " - << globalNLineLets << " lines.\n" - << " Line ends because: " << pct(globalStop[0]) << "% mesh became isotropic, " - << pct(globalStop[1]) << "% no aligned neighbour, " - << pct(globalStop[2]) << "% hit the " << CLineletInfo::MAX_LINELET_POINTS + << " points in each line (average), " << globalMaxNPoints << " longest, " << globalNLineLets + << " lines.\n" + << " Line ends because: " << pct(globalStop[0]) << "% mesh became isotropic, " << pct(globalStop[1]) + << "% no aligned neighbour, " << pct(globalStop[2]) << "% hit the " << CLineletInfo::MAX_LINELET_POINTS << "-point cap.\n" << " of the 'no aligned neighbour': " << pct(globalStop[3]) << "% all candidates already claimed by another line, " << pct(globalStop[4]) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 5d967415b71..03a7a47613d 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1358,9 +1358,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * lockstep makes all lines compete for each layer on equal terms, which on an extruded prismatic * layer reproduces the mesh's own structure: every line reaches the same depth. *================================================================================================*/ - vector> lines; /*!< lines[i] = [wall_node, interior_1, interior_2, ...] */ - vector dir; /*!< Current marching direction of each line, nDim per line. */ - vector claimed(nPointFine, 0); /*!< Node already belongs to some line. */ + vector> lines; /*!< lines[i] = [wall_node, interior_1, interior_2, ...] */ + vector dir; /*!< Current marching direction of each line, nDim per line. */ + vector claimed(nPointFine, 0); /*!< Node already belongs to some line. */ /*--- Nodes that sit on a boundary with a boundary condition on it. A line must not grow into one, * because those nodes belong to the boundary agglomeration and a stack that absorbed one would @@ -1592,6 +1592,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * made the bundles a partition, no two bundles can ever contend for the same node, so a stack is * never interrupted part way up. * + * The lines in a bundle need not be equally long. A stack therefore keeps rising for as long as + * enough of its lines have nodes left, narrowing where the shorter ones end, instead of stopping + * where the shortest one does and abandoning everything the taller ones still had. + * * The multigrid queue is deliberately not updated here: the sync loop that follows the boundary * agglomeration removes every point already marked agglomerated, so removing them a second time * from this function would be an error. @@ -1620,16 +1624,60 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, Index_CoarseCV++; nStacks++; - /*--- The interior layers, in lockstep, to the end of the shortest line in the bundle. ---*/ - unsigned long shortest = ULONG_MAX; - for (auto li : members) shortest = std::min(shortest, lines[li].size()); + /*--- The interior layers, in lockstep. A line that runs out simply stops contributing and the + * others carry on without it, so a bundle is no longer cut down to its shortest member: the + * stack narrows as it rises instead of ending. The CVs still form one connected column, which + * is what a line relaxation on the coarse grid needs. + * + * Two situations end the stack rather than narrowing it further. Dropping below two lines + * would extrude a column one line wide, thinner than anything the domain pass would build + * there, and a set of lines that is no longer connected on the wall would put two separated + * columns into a single CV. In both cases the nodes above are better left to ordinary domain + * agglomeration. A bundle that only ever had one line is exempt from the first rule: it is + * one line wide by construction, and stopping early would gain nothing. ---*/ + + /*--- Are the still-growing lines one connected patch on the wall? Takes positions into members, + * which is at most max_group long, so the quadratic scan over Phase B's adjacency is cheap. ---*/ + auto isConnected = [&adj, &members](const vector& act) { + if (act.size() <= 1) return true; + vector seen(act.size(), 0); + vector stack{0}; + seen[0] = 1; + unsigned long nSeen = 1; + while (!stack.empty()) { + const auto cur = stack.back(); + stack.pop_back(); + const auto& neighbors = adj[members[act[cur]]]; + for (unsigned long k = 0; k < act.size(); ++k) { + if (seen[k]) continue; + if (find(neighbors.begin(), neighbors.end(), members[act[k]]) != neighbors.end()) { + seen[k] = 1; + nSeen++; + stack.push_back(k); + } + } + } + return nSeen == act.size(); + }; + + const unsigned long minActive = std::min(2, members.size()); + + vector placed(members.size(), 1); /*!< First node of each line not yet in a CV. */ + vector active, group; + + for (unsigned long first = 1;; first += nBlock) { + /*--- The lines that still have a whole block left at this height. ---*/ + active.clear(); + for (unsigned long m = 0; m < members.size(); ++m) + if (first + nBlock <= lines[members[m]].size()) active.push_back(m); + + if (active.size() < minActive) break; + if (!isConnected(active)) break; - unsigned long placed = 1; - for (unsigned long first = 1; first + nBlock <= shortest; first += nBlock) { - vector group; - group.reserve(members.size() * nBlock); - for (auto li : members) - for (unsigned long b = 0; b < nBlock; ++b) group.push_back(lines[li][first + b]); + group.clear(); + group.reserve(active.size() * nBlock); + for (auto m : active) + for (unsigned long b = 0; b < nBlock; ++b) group.push_back(lines[members[m]][first + b]); valid = true; for (auto p : group) @@ -1642,12 +1690,11 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } nodes->SetnChildren_CV(Index_CoarseCV, static_cast(group.size())); Index_CoarseCV++; - placed = first + nBlock; + for (auto m : active) placed[m] = first + nBlock; } - /*--- Nodes above the shortest line, or above a block that did not divide evenly, are left to - * ordinary domain agglomeration. ---*/ - for (auto li : members) nTruncated += lines[li].size() - placed; + /*--- Whatever each line still carries above the last CV it contributed to. ---*/ + for (unsigned long m = 0; m < members.size(); ++m) nTruncated += lines[members[m]].size() - placed[m]; } if (DEBUG_OUTPUT) { From b880efa26c907dc2668ee780bc3fbf8cae30416e Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 21 Aug 2026 12:00:25 +0200 Subject: [PATCH 33/54] base implicit line agglomeration on aspect ratio --- Common/include/option_structure.hpp | 4 + Common/src/CConfig.cpp | 5 + Common/src/geometry/CMultiGridGeometry.cpp | 195 +++++++++++++++++++-- 3 files changed, 193 insertions(+), 11 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index ee9cf020ff8..0417e64827f 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1130,6 +1130,10 @@ struct CMGOptions { unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse CV tangential to the wall. 0 = dimension-appropriate default (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ + su2double MG_Implicit_Lines_Min_AR{2.0}; /*!< \brief Smallest local cell aspect ratio for which a node still counts as + part of a stretched layer. Ends a line where the mesh stops being + stretched along it, and decides which boundaries carry a layer normal + to them. See CMultiGridGeometry::AgglomerateImplicitLines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Number of iterations on coarsest mesh during FMG startup phase. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index e63b5eb340e..bc419e94964 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2090,6 +2090,11 @@ void CConfig::SetConfig_Options() { * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_GROUP", MGOptions.MG_Implicit_Lines_Max_Group, 0); + /*!\brief MG_IMPLICIT_LINES_MIN_AR\n DESCRIPTION: Smallest local cell aspect ratio for which a node still counts as part + * of a stretched layer, measured from the ratio of dual-grid edge weights. Ends an implicit line where the mesh stops + * being stretched along it, instead of letting it run to the far field, and decides which boundaries carry a layer + * normal to them and may therefore seed lines. 1.0 disables both tests. DEFAULT: 2.0 \ingroup Config*/ + addDoubleOption("MG_IMPLICIT_LINES_MIN_AR", MGOptions.MG_Implicit_Lines_Min_AR, 2.0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Number of iterations on the coarsest mesh during Full Multigrid (FMG) startup phase before advancing to finer meshes. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); /*!\brief MG_STARTUP_STAGNATION\n DESCRIPTION: Full-MG promotion on stagnation. If the active level's residual ratio diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 03a7a47613d..c056215e17c 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1347,6 +1347,45 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; + /*--- Smallest local cell aspect ratio that still counts as a stretched layer. ---*/ + const su2double MIN_AR = config->GetMGOptions().MG_Implicit_Lines_Min_AR; + const bool USE_AR = (MIN_AR > 1.0); + + /*--- Strength of the coupling across the dual face between a node and one of its neighbours. For a + * cell of streamwise size dx and wall-normal size dy this is 1/dy across the wall-normal face and + * 1/dx across the tangential one, so the ratio of the largest weight at a node to the smallest is + * the local cell aspect ratio. That makes the aspect ratio available from the dual grid alone, + * which SetControlVolume builds on every multigrid level, whereas CGeometry::Aspect_Ratio exists + * only on MESH_0. The same quantity already decides where LINELET preconditioner lines stop, in + * CGeometry::GetLineletInfo. ---*/ + auto edgeWeight = [&](unsigned long iPoint, unsigned short iNeigh) { + const auto jPoint = fine_grid->nodes->GetPoint(iPoint, iNeigh); + const auto iEdge = fine_grid->nodes->GetEdge(iPoint, iNeigh); + const su2double area = GeometryToolbox::Norm(nDim, fine_grid->edges->GetNormal(iEdge)); + return 0.5 * area * (1.0 / fine_grid->nodes->GetVolume(iPoint) + 1.0 / fine_grid->nodes->GetVolume(jPoint)); + }; + + /*--- Weakest coupling at a node, i.e. the denominator of the local aspect ratio. ---*/ + auto minEdgeWeight = [&](unsigned long iPoint) { + su2double wmin = std::numeric_limits::max(); + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) + wmin = std::min(wmin, edgeWeight(iPoint, iNeigh)); + return wmin; + }; + + /*--- Aspect ratio of the mesh at iPoint measured along the edge to jPoint, and the neighbour the + * stiffest edge leads to. Taking the weight of one specific edge over the weakest edge at the + * node, rather than the largest over the smallest, keeps the measure directional: a mesh graded + * in the streamwise direction reads as stretched to the undirected form even far from any wall, + * which is why GetLineletInfo's min/max test cannot be used to decide where a line ends. ---*/ + auto aspectRatioAlong = [&](unsigned long iPoint, unsigned long jPoint) { + const su2double wmin = minEdgeWeight(iPoint); + if (wmin <= 0.0) return su2double(1.0); + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) + if (fine_grid->nodes->GetPoint(iPoint, iNeigh) == jPoint) return edgeWeight(iPoint, iNeigh) / wmin; + return su2double(1.0); + }; + /*================================================================================================== * PHASE A - build the implicit lines. * @@ -1381,10 +1420,28 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } + /*--- Seed a line at iPoint growing away from the boundary along unitNormal. ---*/ + auto seedLine = [&](unsigned long iPoint, const su2double* unitNormal) { + lines.push_back({iPoint}); + claimed[iPoint] = 1; + for (unsigned short d = 0; d < nDim; ++d) dir.push_back(unitNormal[d]); + }; + + /*--- Unit normal of the boundary at a vertex, false if the marker does not reach iPoint. ---*/ + auto vertexNormal = [&](unsigned long iPoint, unsigned short iMarker, su2double* unitNormal) { + const long ChildVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); + if (ChildVertex == -1) return false; + fine_grid->vertex[iMarker][ChildVertex]->GetNormal(unitNormal); + const su2double nrm = GeometryToolbox::Norm(nDim, unitNormal); + if (nrm <= 0.0) return false; + for (unsigned short d = 0; d < nDim; ++d) unitNormal[d] /= nrm; + return true; + }; + + /*--- Viscous walls always carry a stretched layer, so they seed unconditionally. Running them + * first also settles the nodes where a wall meets another boundary: the wall claims them, and + * the line there is a wall line. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - /*--- Seed only from viscous (no-slip) walls: those are the boundaries with a stretched layer - * above them. Seeding from farfield/inlet/outlet/symmetry would claim layer nodes before the - * wall lines could reach them. ---*/ const auto bc = config->GetMarker_All_KindBC(iMarker); if (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL) continue; @@ -1394,16 +1451,123 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; if (claimed[iPoint]) continue; /*--- A node on two wall markers must seed only one line. ---*/ - const long ChildVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); - if (ChildVertex == -1) continue; su2double Normal[MAXNDIM] = {0.0}; - fine_grid->vertex[iMarker][ChildVertex]->GetNormal(Normal); - const su2double nrm = GeometryToolbox::Norm(nDim, Normal); - if (nrm <= 0.0) continue; + if (!vertexNormal(iPoint, iMarker, Normal)) continue; + seedLine(iPoint, Normal); + } + } + + /*================================================================================================== + * Boundaries other than viscous walls that nevertheless carry a stretched layer normal to + * themselves. A symmetry plane laid in the same surface as a wall, such as the one ahead of a + * flat plate's leading edge or the flat floor sections either side of a bump, is meshed with the + * very same normal spacing as the wall it continues. Seeding only from walls leaves the mesh + * above it to isotropic agglomeration, so the coarse grid changes character across the line where + * the two meet even though the fine grid does not, and that shows up as a residual there. + * + * A node qualifies when its stiffest edge is both stretched and points along the boundary normal, + * which is what distinguishes a layer growing off this boundary from one merely passing by: on + * the side planes of a bump the mesh is just as stretched, but in the wall-normal direction that + * runs ALONG the plane, and those nodes belong to the wall's own lines. + * + * The decision is then taken per marker rather than per node. Seeding individual qualifying nodes + * on a marker that mostly does not qualify scatters isolated lines across a face whose neighbours + * seed nothing, and those become one-line bundles, i.e. coarse CVs one fine CV wide that do not + * coarsen tangentially at all. Measured on a flat plate and a 3D bump the two populations are far + * apart - boundaries with a layer normal to them qualify at 100%, while side planes, inlets, + * outlets and far fields come in at 13% and below - so any threshold near a half separates them. + *================================================================================================*/ + if (USE_AR) { + const auto nMarkerFine = fine_grid->GetnMarker(); + + /*--- Counts are kept per marker of the configuration file, not per local marker. Ranks do not + * agree on either the number of markers or their order, because the SEND_RECEIVE markers of a + * partition are appended to its own list, so the same index means a different boundary on + * another rank and a reduction over it would add unrelated boundaries together. The + * configuration file list is the same everywhere. ---*/ + const auto nMarkerCfg = config->GetnMarker_CfgFile(); + vector nValid(nMarkerCfg, 0), nQualified(nMarkerCfg, 0); + + /*--- True if the mesh at iPoint is stretched along the boundary normal, i.e. this boundary has a + * layer growing off it in the same way a viscous wall does. ---*/ + auto hasLayerNormalTo = [&](unsigned long iPoint, const su2double* unitNormal) { + su2double wmax = 0.0, wmin = std::numeric_limits::max(); + unsigned long jStiffest = ULONG_MAX; + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) { + const su2double w = edgeWeight(iPoint, iNeigh); + if (w > wmax) { + wmax = w; + jStiffest = fine_grid->nodes->GetPoint(iPoint, iNeigh); + } + wmin = std::min(wmin, w); + } + if (jStiffest == ULONG_MAX || wmin <= 0.0) return false; + if (wmax / wmin < MIN_AR) return false; + + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jStiffest), fine_grid->nodes->GetCoord(iPoint), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) return false; + for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; + return fabs(GeometryToolbox::DotProduct(nDim, vec, unitNormal)) >= cos_threshold; + }; - lines.push_back({iPoint}); - claimed[iPoint] = 1; - for (unsigned short d = 0; d < nDim; ++d) dir.push_back(Normal[d] / nrm); + /*--- Markers that may be tested at all. Periodic boundaries are left out: the two halves are the + * same physical location under a transform and have their own matching, which a line running + * into one would interfere with. ---*/ + auto canSeed = [&](unsigned short iMarker) { + const auto bc = config->GetMarker_All_KindBC(iMarker); + if (bc == SEND_RECEIVE || bc == PERIODIC_BOUNDARY) return false; + return (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL); + }; + + /*--- Position of a local marker in the configuration file list. Only meaningful for the markers + * canSeed accepts: a SEND_RECEIVE marker is named per partition and is not in that list. ---*/ + vector cfgOfMarker(nMarkerFine, 0); + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) + if (canSeed(iMarker)) + cfgOfMarker[iMarker] = config->GetMarker_CfgFile_TagBound(config->GetMarker_All_TagBound(iMarker)); + + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (!canSeed(iMarker)) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { + const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + su2double Normal[MAXNDIM] = {0.0}; + if (!vertexNormal(iPoint, iMarker, Normal)) continue; + nValid[cfgOfMarker[iMarker]]++; + if (hasLayerNormalTo(iPoint, Normal)) nQualified[cfgOfMarker[iMarker]]++; + } + } + + /*--- A marker is generally split over several ranks, so the verdict has to be taken on the whole + * of it or two ranks could disagree about the same boundary. ---*/ + if (nMarkerCfg > 0) { + vector tmp(nMarkerCfg); + SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + nValid.swap(tmp); + SU2_MPI::Allreduce(nQualified.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + nQualified.swap(tmp); + } + + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (!canSeed(iMarker)) continue; + const auto iCfg = cfgOfMarker[iMarker]; + if (nValid[iCfg] == 0) continue; + if (2 * nQualified[iCfg] < nValid[iCfg]) continue; /*--- Fewer than half, not a layer. ---*/ + + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { + const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; + if (claimed[iPoint]) continue; + + su2double Normal[MAXNDIM] = {0.0}; + if (!vertexNormal(iPoint, iMarker, Normal)) continue; + /*--- The marker carries a layer, but this node still has to be in it. ---*/ + if (!hasLayerNormalTo(iPoint, Normal)) continue; + seedLine(iPoint, Normal); + } } } @@ -1450,6 +1614,15 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, continue; } + /*--- End the line where the mesh stops being stretched along it. Without this the only limits + * are the direction cone and MAX_LINE_LENGTH, so a line leaves the boundary layer and keeps + * going into the far field, stacking coarse CVs along a direction the fine grid does not + * single out. Ordinary agglomeration handles that region better. ---*/ + if (USE_AR && aspectRatioAlong(current, best_neighbor) < MIN_AR) { + growing[li] = 0; + continue; + } + su2double step[MAXNDIM] = {0.0}; GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(best_neighbor), fine_grid->nodes->GetCoord(current), step); From 7be5d3d6e221ec9ed506e3a988e781f092357438 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 21 Aug 2026 15:17:18 +0200 Subject: [PATCH 34/54] change parmetis anisotropy --- Common/include/CConfig.hpp | 6 + Common/src/CConfig.cpp | 8 ++ Common/src/geometry/CPhysicalGeometry.cpp | 151 +++++++++++++++++++++- 3 files changed, 161 insertions(+), 4 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 9605b0a566e..dbfcb06f7b5 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -1099,6 +1099,7 @@ class CConfig { su2double ParMETIS_tolerance; /*!< \brief Load balancing tolerance for ParMETIS. */ long ParMETIS_pointWgt; /*!< \brief Load balancing weight given to points. */ long ParMETIS_edgeWgt; /*!< \brief Load balancing weight given to edges. */ + su2double ParMETIS_anisoWgt; /*!< \brief Strength of the anisotropy-aware ParMETIS edge weights. 0 disables them. */ unsigned short DirectDiff; /*!< \brief Direct Differentation mode. */ bool DiscreteAdjoint, /*!< \brief AD-based discrete adjoint mode. */ DiscreteAdjointDebug; /*!< \brief Discrete adjoint debug mode using tags. */ @@ -10184,6 +10185,11 @@ class CConfig { */ long GetParMETIS_EdgeWeight() const { return ParMETIS_edgeWgt; } + /*! + * \brief Get the strength of the anisotropy-aware ParMETIS edge weights (0 disables them). + */ + passivedouble GetParMETIS_AnisoWeight() const { return SU2_TYPE::GetValue(ParMETIS_anisoWgt); } + /*! * \brief Find the marker index (if any) that is part of a given interface pair. * \param[in] iInterface - Number of the interface pair being tested, starting at 0. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index bc419e94964..32e8a02841f 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -3073,6 +3073,14 @@ void CConfig::SetConfig_Options() { /* DESCRIPTION: ParMETIS load balancing weight for edges (equiv. to neighbors) */ addLongOption("PARMETIS_EDGE_WEIGHT", ParMETIS_edgeWgt, 1); + /* DESCRIPTION: Strength of the anisotropy-aware ParMETIS edge weights. ParMETIS is otherwise given no edge weights at + * all, so every edge is equally cheap to cut and partition boundaries slice straight through the stretched cells of a + * boundary layer, splitting the wall-normal columns that implicit-line agglomeration and line-implicit smoothing rely + * on. Weighting an edge by the inverse of its length makes the short wall-normal edges expensive to cut and pushes the + * cuts into the tangential direction instead. On a mesh without stretching all edges are of similar length, the + * weights come out uniform, and the partitioning is the same as with no weights at all. 0 disables the weights. + * DEFAULT: 0 */ + addDoubleOption("PARMETIS_ANISO_WEIGHT", ParMETIS_anisoWgt, 0.0); /*--- options that are used in the Hybrid RANS/LES Simulations ---*/ /*!\par CONFIG_CATEGORY:Hybrid_RANSLES Options\ingroup Config*/ diff --git a/Common/src/geometry/CPhysicalGeometry.cpp b/Common/src/geometry/CPhysicalGeometry.cpp index 199c8cec03f..486701d99c7 100644 --- a/Common/src/geometry/CPhysicalGeometry.cpp +++ b/Common/src/geometry/CPhysicalGeometry.cpp @@ -7377,7 +7377,7 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { /*--- Some recommended defaults for the various ParMETIS options. ---*/ - idx_t wgtflag = 2; + idx_t wgtflag = 2; /*--- Weights on the vertices only, raised to 3 below if edge weights are built. ---*/ idx_t numflag = 0; idx_t ncon = 1; real_t ubvec = 1.0 + config->GetParMETIS_Tolerance(); @@ -7410,6 +7410,149 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { vwgt[iPoint] = wp + we * (xadj[iPoint + 1] - xadj[iPoint]); } + /*--- Cost of cutting each edge of the graph. + * + * Without these ParMETIS is given no edge weights at all and every edge is equally cheap to + * cut, so nothing stops a partition boundary from running straight through the stretched cells + * of a boundary layer and splitting the wall-normal columns that implicit-line agglomeration + * and line-implicit smoothing are built on. + * + * In a stretched cell the wall-normal spacing is the small one, so the short edges are exactly + * the ones that should stay inside a partition. Weighting an edge by the inverse of its length + * therefore makes cutting across the layer expensive and leaves the long tangential edges as + * the cheap place to cut. Length is used rather than the face area over volume ratio that + * measures the same thing elsewhere, because the dual grid does not exist yet at this point of + * the setup: this runs before SetControlVolume, and only the coordinates are available. + * + * On a mesh with no stretching every edge is of similar length, so the weights come out + * uniform and minimizing their sum is the same problem as minimizing the number of cut edges. + * Such a mesh is therefore partitioned exactly as it is with no weights at all. ---*/ + + vector adjwgt; + const su2double anisoWgt = config->GetParMETIS_AnisoWeight(); + + if (anisoWgt > 0.0) { + const auto firstIdx = pointPartitioner.GetFirstIndexOnRank(rank); + const auto lastIdx = pointPartitioner.GetLastIndexOnRank(rank); + auto isLocal = [&](unsigned long g) { return g >= firstIdx && g < lastIdx; }; + + /*--- The graph is split linearly and its entries are global indices, so an edge near a linear + * partition boundary has one end that is not stored here. Those are few, of the order of a + * percent of the entries, and each is asked for from the rank the linear partitioner says + * owns it. The same request lists are used twice, once for coordinates and once for the + * longest edge at the point, which only its owner can work out. ---*/ + vector> wanted(size); + for (auto gPoint : adjacency) + if (!isLocal(gPoint)) wanted[pointPartitioner.GetRankContainingIndex(gPoint)].push_back(gPoint); + + vector nSend(size, 0), nRecv(size, 0), sDisp(size + 1, 0), rDisp(size + 1, 0); + for (int r = 0; r < size; ++r) { + auto& w = wanted[r]; + sort(w.begin(), w.end()); + w.erase(unique(w.begin(), w.end()), w.end()); + nSend[r] = static_cast(w.size()); + } + SU2_MPI::Alltoall(nSend.data(), 1, MPI_INT, nRecv.data(), 1, MPI_INT, comm); + for (int r = 0; r < size; ++r) { + sDisp[r + 1] = sDisp[r] + nSend[r]; + rDisp[r + 1] = rDisp[r] + nRecv[r]; + } + + vector sendIdx(sDisp[size]), recvIdx(rDisp[size]); + for (int r = 0; r < size; ++r) copy(wanted[r].begin(), wanted[r].end(), sendIdx.begin() + sDisp[r]); + SU2_MPI::Alltoallv(sendIdx.data(), nSend.data(), sDisp.data(), MPI_UNSIGNED_LONG, recvIdx.data(), nRecv.data(), + rDisp.data(), MPI_UNSIGNED_LONG, comm); + + /*--- Round one, the coordinates of the points that were asked for. ---*/ + map> remoteCoord; + { + vector sendBuf(static_cast(rDisp[size]) * nDim), + recvBuf(static_cast(sDisp[size]) * nDim); + for (size_t i = 0; i < recvIdx.size(); ++i) + for (unsigned short iDim = 0; iDim < nDim; ++iDim) + sendBuf[i * nDim + iDim] = nodes->GetCoord(recvIdx[i] - firstIdx, iDim); + + vector nS(size), nR(size), sD(size), rD(size); + for (int r = 0; r < size; ++r) { + nS[r] = nRecv[r] * nDim; + nR[r] = nSend[r] * nDim; + sD[r] = rDisp[r] * nDim; + rD[r] = sDisp[r] * nDim; + } + SU2_MPI::Alltoallv(sendBuf.data(), nS.data(), sD.data(), MPI_DOUBLE, recvBuf.data(), nR.data(), rD.data(), + MPI_DOUBLE, comm); + for (size_t i = 0; i < sendIdx.size(); ++i) { + array c = {0.0, 0.0, 0.0}; + for (unsigned short iDim = 0; iDim < nDim; ++iDim) c[iDim] = recvBuf[i * nDim + iDim]; + remoteCoord[sendIdx[i]] = c; + } + } + + /*--- Length of every edge of the local part of the graph, and the longest edge at each point + * this rank owns. ---*/ + vector edgeLen(adjacency.size(), 0.0); + vector maxLen(nPoint, 0.0); + + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { + const auto gPoint = adjacency[k]; + array tmp = {0.0, 0.0, 0.0}; + const su2double* coord_j = nullptr; + if (isLocal(gPoint)) { + coord_j = nodes->GetCoord(gPoint - firstIdx); + } else { + const auto it = remoteCoord.find(gPoint); + if (it == remoteCoord.end()) continue; + tmp = it->second; + coord_j = tmp.data(); + } + edgeLen[k] = GeometryToolbox::Distance(nDim, nodes->GetCoord(iPoint), coord_j); + maxLen[iPoint] = max(maxLen[iPoint], edgeLen[k]); + } + } + + /*--- Round two, the longest edge at each of the remote points. ---*/ + map remoteMaxLen; + { + vector sendBuf(rDisp[size]), recvBuf(sDisp[size]); + for (size_t i = 0; i < recvIdx.size(); ++i) sendBuf[i] = maxLen[recvIdx[i] - firstIdx]; + SU2_MPI::Alltoallv(sendBuf.data(), nRecv.data(), rDisp.data(), MPI_DOUBLE, recvBuf.data(), nSend.data(), + sDisp.data(), MPI_DOUBLE, comm); + for (size_t i = 0; i < sendIdx.size(); ++i) remoteMaxLen[sendIdx[i]] = recvBuf[i]; + } + + /*--- An edge is expensive to cut when it is much shorter than the other edges meeting it, which + * is the definition of the local cell aspect ratio and is exactly the situation inside a + * boundary layer, where the short edges are the wall-normal ones. Comparing an edge only + * against its own neighbourhood, rather than against a global length, is what keeps the + * measure a ratio: scaling the whole mesh, or refining one region of it isotropically, leaves + * every weight unchanged. Weighting by absolute length instead would make any small cell + * expensive to cut and would steer the partitioner away from refined regions that are not + * stretched at all. Averaging the two ends keeps the weight of an edge symmetric, which + * ParMETIS requires. ---*/ + const idx_t MAX_EDGE_WEIGHT = 1000; + adjwgt.resize(adjacency.size(), 1); + + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { + if (edgeLen[k] <= 0.0) continue; + const auto gPoint = adjacency[k]; + su2double maxLen_j = 0.0; + if (isLocal(gPoint)) { + maxLen_j = maxLen[gPoint - firstIdx]; + } else { + const auto it = remoteMaxLen.find(gPoint); + if (it == remoteMaxLen.end()) continue; + maxLen_j = it->second; + } + const su2double ratio = 0.5 * (maxLen[iPoint] + maxLen_j) / edgeLen[k]; + const su2double w = 1.0 + anisoWgt * (ratio - 1.0); + adjwgt[k] = static_cast(min(max(w, 1.0), MAX_EDGE_WEIGHT)); + } + } + wgtflag = 3; /*--- Weights on both the vertices and the edges. ---*/ + } + /*--- Create some structures that ParMETIS needs to output the partitioning. ---*/ idx_t edgecut; @@ -7418,9 +7561,9 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { /*--- Calling ParMETIS ---*/ if (rank == MASTER_NODE) cout << "Calling ParMETIS..."; - auto err = - ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), nullptr, &wgtflag, &numflag, - &ncon, &nparts, tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); + auto err = ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), + adjwgt.empty() ? nullptr : adjwgt.data(), &wgtflag, &numflag, &ncon, &nparts, + tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); if (err != METIS_OK) SU2_MPI::Error("Partitioning failed.", CURRENT_FUNCTION); if (rank == MASTER_NODE) { cout << " graph partitioning complete (" << edgecut << " edge cuts)." << endl; From 192c3e1509c715a600921ef03444fdac4feee751 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 21 Aug 2026 18:00:22 +0200 Subject: [PATCH 35/54] fix some openmp stuff --- .../src/integration/CMultiGridIntegration.cpp | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index f8bd975b3c9..4665c295d9f 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -930,6 +930,10 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ const unsigned short nVar = solver->GetnVar(); + /*--- Seeded over all points, halos included: the restore loop below reads Residual_Old at the + * vertices of the physical markers, and on a partitioned mesh some of those are halo points + * owned by another rank. ---*/ + SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); iPoint++) { const auto* Residual_Old = solver->LinSysRes.GetBlock(iPoint); @@ -942,10 +946,13 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ for (auto iSmooth = 0u; iSmooth < val_nSmooth; iSmooth++) { - /*--- Loop over all mesh points (sum the residuals of direct neighbors). ---*/ + /*--- Loop over the domain points (sum the residuals of direct neighbors). + * Halo points are deliberately not smoothed here: their own neighbor stencil is incomplete + * on this rank, so the average would be meaningless, and the halo exchange at the end of + * each sweep overwrites them with the value their owner computed anyway. ---*/ - SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) - for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); ++iPoint) { + SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPointDomain(), omp_get_num_threads())) + for (auto iPoint = 0ul; iPoint < geometry->GetnPointDomain(); ++iPoint) { solver->GetNodes()->SetResidualSumZero(iPoint); @@ -958,10 +965,10 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } END_SU2_OMP_FOR - /*--- Loop over all mesh points (update residuals with the neighbor averages). ---*/ + /*--- Loop over the domain points (update residuals with the neighbor averages). ---*/ - SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) - for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); ++iPoint) { + SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPointDomain(), omp_get_num_threads())) + for (auto iPoint = 0ul; iPoint < geometry->GetnPointDomain(); ++iPoint) { su2double factor = 1.0/(1.0+val_smooth_coeff*su2double(geometry->nodes->GetnPoint(iPoint))); @@ -973,20 +980,23 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } END_SU2_OMP_FOR - /*--- Restore original residuals (without average) at boundary points. + /*--- Restore original residuals (without average) at physical boundary points. + * + * SEND_RECEIVE is excluded: carrying such a marker does not put a point on a boundary, it + * only records that the point is mirrored on another rank. Restoring those points froze the + * correction on the whole send fringe, which is exactly the ring of domain points that have + * a halo neighbour, so the smoothing this function applied depended on where the mesh + * happened to be partitioned rather than on the geometry alone. * - * FIXME (MPI): SEND_RECEIVE is not excluded here, so every point on a partition interface - * has its smoothed correction reverted after each sweep. That is why this smoother gives a - * different convergence history on 1 and on N ranks. Excluding SEND_RECEIVE is only half the - * fix: the sweeps also read LinSysRes at halo points, which ProlongateField fills once but - * nothing refreshes between sweeps, so a halo exchange of LinSysRes is needed inside the - * loop (there is no MPI_QUANTITIES entry for it yet). Until both are done this smoother is - * only parallel-consistent for MG_CORRECTION_SMOOTH= 0, which is the default. ---*/ + * Note this removes one source of rank-dependence, not all of them: the coarse grids are + * agglomerated per rank, so the multigrid operator itself still differs between partition + * counts and a run on 1 and on N ranks is not expected to match bit for bit. ---*/ for (auto iMarker = 0u; iMarker < geometry->GetnMarker(); iMarker++) { if ((config->GetMarker_All_KindBC(iMarker) != INTERNAL_BOUNDARY) && (config->GetMarker_All_KindBC(iMarker) != NEARFIELD_BOUNDARY) && - (config->GetMarker_All_KindBC(iMarker) != PERIODIC_BOUNDARY)) { + (config->GetMarker_All_KindBC(iMarker) != PERIODIC_BOUNDARY) && + (config->GetMarker_All_KindBC(iMarker) != SEND_RECEIVE)) { SU2_OMP_FOR_STAT(32) for (auto iVertex = 0ul; iVertex < geometry->GetnVertex(iMarker); iVertex++) { @@ -998,6 +1008,19 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } } + /*--- Refresh the halo entries of the correction with the values their owner ranks just + * computed. The next sweep averages LinSysRes over the neighbours of every domain point, + * and across a partition boundary those neighbours are halo points, so this has to run + * once per sweep rather than once at the end. It comes after the restore so that a halo + * point sitting on a physical boundary mirrors its owner's restored value. + * + * The barrier is required: the restore loop above only carries an implicit barrier for the + * markers that pass the test, so if the last marker is skipped there is none. ---*/ + + SU2_OMP_BARRIER + CSysMatrixComms::Initiate(solver->LinSysRes, geometry, config); + CSysMatrixComms::Complete(solver->LinSysRes, geometry, config); + } /*--- Record final correction norm for debugging output. ---*/ From 0931e1475c2093422324274e97113c9112eaec91 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 30 Aug 2026 12:57:45 +0200 Subject: [PATCH 36/54] cleanup --- .../include/geometry/CMultiGridGeometry.hpp | 50 +- Common/include/option_structure.hpp | 5 + Common/src/CConfig.cpp | 6 + Common/src/geometry/CMultiGridGeometry.cpp | 992 ++++++++++-------- 4 files changed, 628 insertions(+), 425 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index ba635b40f8c..2dd83b0fd0e 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -82,10 +82,54 @@ class CMultiGridGeometry final : public CGeometry { * \param[in,out] Index_CoarseCV - Current coarse CV index, incremented as new coarse CVs are created. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Configuration. - * \param[in,out] MGQueue_InnerCV - Queue for domain agglomeration; processed points are removed. */ - void AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, - CMultiGridQueue& MGQueue_InnerCV); + void AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config); + + /*! + * \brief Per-node dual-grid stiffness data used by the implicit-line agglomeration: the weakest and + * strongest coupling at each node, and the neighbour the strongest one leads to. Their ratio is + * the local cell aspect ratio, available on every multigrid level unlike CGeometry::Aspect_Ratio. + */ + struct CNodeStiffness { + vector wMin, wMax; /*!< \brief Weakest and strongest edge coupling at each node. */ + vector jStiffest; /*!< \brief Neighbour across the strongest edge. */ + + /*!< \brief Local aspect ratio at a node, 1.0 where it could not be measured. */ + su2double AspectRatio(unsigned long iPoint) const { + return (wMin[iPoint] > 0.0) ? wMax[iPoint] / wMin[iPoint] : su2double(1.0); + } + }; + + /*! + * \brief Measure the dual-grid coupling at every node of a grid. + * \param[in] fine_grid - Grid to measure. + * \return Weakest/strongest coupling per node, see CNodeStiffness. + */ + CNodeStiffness ComputeNodeStiffness(const CGeometry* fine_grid) const; + + /*! + * \brief PHASE A of the implicit-line agglomeration: grow wall-normal lines of nodes from the + * boundaries that carry a stretched layer. Lines are node-disjoint. + * \param[in] fine_grid - Fine grid geometry. + * \param[in] config - Configuration. + * \param[in] stiff - Node coupling from ComputeNodeStiffness. + * \return One vector per line, each starting at its boundary node. + */ + vector> BuildImplicitLines(const CGeometry* fine_grid, const CConfig* config, + const CNodeStiffness& stiff) const; + + /*! + * \brief PHASE B of the implicit-line agglomeration: partition the lines into compact bundles that + * share a wall footprint, by repeated pairwise matching. + * \param[in] lines - Lines from BuildImplicitLines. + * \param[in] fine_grid - Fine grid geometry. + * \param[in] config - Configuration. + * \param[out] adj - Line adjacency inherited from the boundary nodes, reused by PHASE C. + * \return One vector of line indices per bundle. + */ + vector> BundleImplicitLines(const vector>& lines, + const CGeometry* fine_grid, const CConfig* config, + vector>& adj) const; public: /*--- This is to suppress Woverloaded-virtual, omitting it has no negative impact. ---*/ diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index bcc766f08da..22834686447 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1134,6 +1134,11 @@ struct CMGOptions { part of a stretched layer. Ends a line where the mesh stops being stretched along it, and decides which boundaries carry a layer normal to them. See CMultiGridGeometry::AgglomerateImplicitLines. */ + su2double MG_Implicit_Lines_Iso_AR{0.0}; /*!< \brief Aspect ratio below which a stack switches from semi-coarsening + (one fine layer per coarse CV, preserving the line) to full coarsening + (two fine layers per coarse CV). 0 disables the switch, leaving + MG_IMPLICIT_LINES_ISOTROPIC in charge for the whole stack. + See CMultiGridGeometry::AgglomerateImplicitLines. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 23273086cb4..6b7aca7fbc6 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2095,6 +2095,12 @@ void CConfig::SetConfig_Options() { * being stretched along it, instead of letting it run to the far field, and decides which boundaries carry a layer * normal to them and may therefore seed lines. 1.0 disables both tests. DEFAULT: 2.0 \ingroup Config*/ addDoubleOption("MG_IMPLICIT_LINES_MIN_AR", MGOptions.MG_Implicit_Lines_Min_AR, 2.0); + /*!\brief MG_IMPLICIT_LINES_ISO_AR\n DESCRIPTION: Local cell aspect ratio at which a stack of coarse CVs switches from + * semi-coarsening to full coarsening. While the mesh at the current height is stretched by more than this, one fine + * layer goes into each coarse CV, so the coarse grid keeps the wall-normal line structure a line-implicit smoother + * relies on. Once the ratio drops below it the layers are taken two at a time and the coarsening becomes isotropic. + * 0.0 disables the switch and MG_IMPLICIT_LINES_ISOTROPIC decides for the whole stack. DEFAULT: 0.0 \ingroup Config*/ + addDoubleOption("MG_IMPLICIT_LINES_ISO_AR", MGOptions.MG_Implicit_Lines_Iso_AR, 0.0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index c056215e17c..98172777442 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -95,7 +95,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * stack above it could then only be misaligned with its own base. Everything it claims is * already marked agglomerated, so the boundary and interior passes below simply skip it. ---*/ if (config->GetMGOptions().MG_Implicit_Lines) { - AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, MGQueue_InnerCV); + AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config); } /*--- STEP 1: The first step is the boundary agglomeration. ---*/ @@ -336,7 +336,35 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- STEP 2: Agglomerate the domain points. ---*/ + /*--- STEP 2: Agglomerate the domain points. + * + * A seed is grown one node at a time, and at every step the node that joins is the one most tied + * to what the CV already holds, i.e. the candidate sharing the most edges with its current members. + * Taking the seed's neighbours in whatever order the connectivity happens to list them, as this + * used to, ignores the shape of the result: on a hex mesh the seed has one neighbour per axis, so + * the first few are picked from different axes and the CV grows into a star. The leftovers around + * it then have to be swept up by later seeds, which is where the ragged pieces come from - a CV + * holding three nodes in one mesh plane and a single node in the next reads, in a cross-section + * through the second plane, as an isolated node sitting in the corner of an L. + * + * Counting shared edges instead closes those shapes off by construction. Once a seed has taken two + * nodes along different axes, the node diagonally between them touches two members while every + * other option still touches one, so it wins and completes the square; the same argument then + * repeats one axis up and completes the cube. On a structured hex mesh the result is an exact + * 2x2x2 block, which is what the implicit-line stacks already produce and what the isotropic + * region should match. Ties are settled by distance to the centroid of the current members, which + * keeps the growth compact on meshes with no preferred axis to reason about. + * + * The candidate set grows with the CV rather than being fixed to the seed's own neighbours: the + * far corner of a cube is not adjacent to the seed, it only becomes reachable once the nodes + * between them have joined. That also makes SetSuitableNeighbors unnecessary here, since the nodes + * it used to supply are reached through ordinary edges as the frontier advances. ---*/ + + /*--- Scratch shared by all seeds. The markers are cleared per CV, touching only what was used. ---*/ + vector inCV(fine_grid->GetnPoint(), 0); + vector isCandidate(fine_grid->GetnPoint(), 0); + vector members, candidates; + members.reserve(maxAgglomSize); auto iteration = 0ul; while (!MGQueue_InnerCV.EmptyQueue() && (iteration < fine_grid->GetnPoint())) { @@ -348,79 +376,70 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if ((!fine_grid->nodes->GetAgglomerate(iPoint)) && (fine_grid->nodes->GetDomain(iPoint)) && (GeometricalCheck(iPoint, fine_grid, config))) { - unsigned short nChildren = 1; - - /*--- We set an index for the parent control volume ---*/ - - fine_grid->nodes->SetParent_CV(iPoint, Index_CoarseCV); - - /*--- We add the seed point (child) to the parent control volume ---*/ - - nodes->SetChildren_CV(Index_CoarseCV, 0, iPoint); - - /*--- Update the queue with the seed point (remove the seed and - increase the priority of its neighbors) ---*/ - - MGQueue_InnerCV.Update(iPoint, fine_grid); - - /*--- Now we do a sweep over all the nodes that surround the seed point ---*/ - - for (auto CVPoint : fine_grid->nodes->GetPoints(iPoint)) { - /*--- Determine if the CVPoint can be agglomerated ---*/ - - if ((!fine_grid->nodes->GetAgglomerate(CVPoint)) && (fine_grid->nodes->GetDomain(CVPoint)) && - (GeometricalCheck(CVPoint, fine_grid, config))) { - /*--- We set the value of the parent ---*/ - - fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); + members.clear(); + candidates.clear(); + unsigned short nChildren = 0; + + /*--- Take a node into the CV and let the frontier grow with it. ---*/ + auto addMember = [&](unsigned long CVPoint) { + fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); + nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); + nChildren++; - /*--- We set the value of the child ---*/ + if (fine_grid->nodes->GetAgglomerate_Indirect(CVPoint)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); - nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); - nChildren++; + /*--- Remove it from the queue and raise the priority of its neighbours. ---*/ + MGQueue_InnerCV.Update(CVPoint, fine_grid); - /*--- Update the queue with the new control volume (remove the CV and - increase the priority of its neighbors) ---*/ + members.push_back(CVPoint); + inCV[CVPoint] = 1; - MGQueue_InnerCV.Update(CVPoint, fine_grid); + for (auto jPoint : fine_grid->nodes->GetPoints(CVPoint)) { + if (inCV[jPoint] || isCandidate[jPoint]) continue; + if (fine_grid->nodes->GetAgglomerate(jPoint) || !fine_grid->nodes->GetDomain(jPoint)) continue; + if (!GeometricalCheck(jPoint, fine_grid, config)) continue; + isCandidate[jPoint] = 1; + candidates.push_back(jPoint); } - if (nChildren == maxAgglomSize) break; - } - - /*--- Identify the indirect neighbors ---*/ - - Suitable_Indirect_Neighbors.clear(); - if (fine_grid->nodes->GetAgglomerate_Indirect(iPoint)) - SetSuitableNeighbors(Suitable_Indirect_Neighbors, iPoint, Index_CoarseCV, fine_grid); - - /*--- Now we do a sweep over all the indirect nodes that can be added ---*/ + }; - for (auto CVPoint : Suitable_Indirect_Neighbors) { - // if we have reached the maximum, get out. - if (nChildren == maxAgglomSize) break; - /*--- The new point can be agglomerated ---*/ + addMember(iPoint); - if ((!fine_grid->nodes->GetAgglomerate(CVPoint)) && (fine_grid->nodes->GetDomain(CVPoint))) { - /*--- We set the value of the parent ---*/ - - fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); - - /*--- We set the indirect agglomeration information ---*/ + while (nChildren < maxAgglomSize) { + /*--- Centroid of what the CV holds so far, used only to break ties. ---*/ + su2double centroid[MAXNDIM] = {0.0}; + for (auto jPoint : members) { + const auto* coord = fine_grid->nodes->GetCoord(jPoint); + for (auto iDim = 0u; iDim < nDim; iDim++) centroid[iDim] += coord[iDim] / su2double(members.size()); + } - if (fine_grid->nodes->GetAgglomerate_Indirect(CVPoint)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); + unsigned long best = std::numeric_limits::max(); + unsigned short best_shared = 0; + su2double best_dist = std::numeric_limits::max(); - /*--- We set the value of the child ---*/ + for (auto CVPoint : candidates) { + if (inCV[CVPoint]) continue; - nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); - nChildren++; + unsigned short shared = 0; + for (auto jPoint : fine_grid->nodes->GetPoints(CVPoint)) shared += inCV[jPoint]; - /*--- Update the queue with the new control volume (remove the CV and - increase the priority of the neighbors) ---*/ + const su2double dist = + GeometryToolbox::SquaredDistance(nDim, fine_grid->nodes->GetCoord(CVPoint), centroid); - MGQueue_InnerCV.Update(CVPoint, fine_grid); + if ((shared > best_shared) || ((shared == best_shared) && (dist < best_dist))) { + best = CVPoint; + best_shared = shared; + best_dist = dist; + } } + + if (best == std::numeric_limits::max()) break; + addMember(best); } + for (auto jPoint : members) inCV[jPoint] = 0; + for (auto jPoint : candidates) isCandidate[jPoint] = 0; + /*--- Update the number of control of childrens ---*/ nodes->SetnChildren_CV(Index_CoarseCV, nChildren); @@ -543,6 +562,109 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- The pass above only rescues a coarse CV that has a single coarse neighbor, i.e. one enclosed + entirely within another. It does nothing for a CV that was left with a single fine-grid child but + sits between several coarse neighbors, which happens routinely with the implicit-line stacks above: + a bundle whose members reach the top of the boundary layer at slightly different heights (graded + spacing, local curvature) narrows one line at a time (see AgglomerateImplicitLines, Phase C), and + the fine node just past where a line dropped out is left to fend for itself. It usually has no + unclaimed neighbor left to agglomerate with by the time the ordinary domain pass (STEP 2) reaches + it, so it becomes a coarse CV of its own, surrounded on multiple sides by CVs it cannot join under + the single-neighbor rule above. Left alone, this is exactly the "hole" that turns what should be a + compact block into an L shape with an isolated singleton sitting in the missing corner. Merge such + a CV into whichever neighbor currently has the fewest children (and still has room under + maxAgglomSize): the smallest neighbor is the one most likely to be the under-filled block the + singleton belongs with, e.g. the 3-node L that this merge completes into a 4-node square, and + folding into it keeps the coarse grid from accumulating disproportionately large CVs. + + Unlike the pass above this one deliberately does not skip CVs that border another rank. That guard + exists because "has exactly one coarse neighbour" is measured on connectivity this rank cannot yet + see in full, so it misfires on a CV whose other neighbours merely live across the partition. The + trigger here is a child count, which is complete locally: a CV owning one fine point owns one fine + point no matter how the mesh was cut. The merge is local too - every candidate neighbour comes from + the connectivity built above and is therefore an owned CV, every child is an owned point, and the + MPI relay below broadcasts the result afterwards, so the ranks stay consistent. Requiring at least + two local neighbours below also rules out the one case that could not be repaired locally, a CV + whose neighbours are all halo: a halo CV mirrors another rank's decision and must not be added to + here. Keeping the guard cost 455 of 473 unrepaired singletons on a four-rank 3D bump. ---*/ + + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { + if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; + if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ + + /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the + smallest also means a neighbour with room is always preferred over a full one: anything below + maxAgglomSize necessarily has fewer children than anything at it. When every neighbour is full + the smallest is taken anyway, one child over the limit. maxAgglomSize is a quality target rather + than a capacity - Children_CV grows on demand - and a CV of nine is a far smaller blemish on the + coarse grid than the CV of one it removes. ---*/ + unsigned long best_neighbor = std::numeric_limits::max(); + unsigned short best_nChildren = 0; + for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { + const auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); + /*--- Skip neighbors already emptied by an earlier merge in this same pass. ---*/ + if (nChildren_Neighbor == 0) continue; + if ((best_neighbor == std::numeric_limits::max()) || (nChildren_Neighbor < best_nChildren)) { + best_nChildren = nChildren_Neighbor; + best_neighbor = jCoarsePoint; + } + } + if (best_neighbor == std::numeric_limits::max()) continue; /*--- Every neighbor was emptied. ---*/ + + const auto iFinePoint = nodes->GetChildren_CV(iCoarsePoint, 0); + nodes->SetChildren_CV(best_neighbor, best_nChildren, iFinePoint); + nodes->SetnChildren_CV(best_neighbor, best_nChildren + 1); + fine_grid->nodes->SetParent_CV(iFinePoint, best_neighbor); + nodes->SetnChildren_CV(iCoarsePoint, 0); + } + + /*--- Compact the coarse numbering. Both repair passes empty a control volume by moving its children + elsewhere, but nPointDomain was fixed before them, so an emptied index survives as a control volume + that owns no fine points at all. It ends up with EPS volume, (0,0,0) coordinates and, because + neighbours are derived through children, no neighbours either. Nothing is then adjacent to it on the + next level, so it cannot be agglomerated there and becomes a one-child CV again, and so on down the + hierarchy: the repair would otherwise trade a bad CV on this level for a degenerate one on every + level above. Squeezing the empty slots out keeps the CV count honest and stops that propagation. + + Renumbering is safe here because only three things refer to coarse indices at this point - the + children lists, the indirect-agglomeration flags, and the fine grid's parent indices - and the MPI + relay below has not run yet, so no other rank has seen these numbers. Only parents of *owned* fine + points are remapped; halo points are assigned by the relay afterwards. Every surviving parent keeps + a valid new index because a CV that still has a child is never removed. ---*/ + + { + constexpr auto NO_INDEX = std::numeric_limits::max(); + vector newIndex(nPointDomain, NO_INDEX); + unsigned long nKept = 0; + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) + if (nodes->GetnChildren_CV(iCoarsePoint) > 0) newIndex[iCoarsePoint] = nKept++; + + if (nKept < nPointDomain) { + /*--- Move the survivors down. newIndex[i] <= i, so a forward sweep never lands on a CV that has + not been moved yet. ---*/ + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { + const auto iNew = newIndex[iCoarsePoint]; + if ((iNew == NO_INDEX) || (iNew == iCoarsePoint)) continue; + const auto nChildren = nodes->GetnChildren_CV(iCoarsePoint); + for (auto iChildren = 0u; iChildren < nChildren; iChildren++) + nodes->SetChildren_CV(iNew, iChildren, nodes->GetChildren_CV(iCoarsePoint, iChildren)); + nodes->SetnChildren_CV(iNew, nChildren); + nodes->SetAgglomerate_Indirect(iNew, nodes->GetAgglomerate_Indirect(iCoarsePoint)); + } + for (auto iCoarsePoint = nKept; iCoarsePoint < nPointDomain; iCoarsePoint++) + nodes->SetnChildren_CV(iCoarsePoint, 0); + + for (auto iFinePoint = 0ul; iFinePoint < fine_grid->GetnPointDomain(); iFinePoint++) { + const auto iParent = fine_grid->nodes->GetParent_CV(iFinePoint); + if (iParent != NO_INDEX) fine_grid->nodes->SetParent_CV(iFinePoint, newIndex[iParent]); + } + + nPointDomain = nKept; + nPoint = nKept; + Index_CoarseCV = nKept; + } + } + /*--- Reset the neighbor information. ---*/ nodes->ResetPoints(); @@ -1329,105 +1451,88 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, return max_angle; } -void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, - const CConfig* config, CMultiGridQueue& MGQueue_InnerCV) { - /*--- Parameters ---*/ - const su2double ANGLE_THRESHOLD_DEG = 20.0; /*!< Stop a line if the direction deviates more than this. */ - const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; - const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); - const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; +CMultiGridGeometry::CNodeStiffness CMultiGridGeometry::ComputeNodeStiffness(const CGeometry* fine_grid) const { + /*--- Strength of the coupling across the dual face between a node and one of its neighbours. For a cell + * of streamwise size dx and wall-normal size dy this is 1/dy across the wall-normal face and 1/dx + * across the tangential one, so the ratio of the largest weight at a node to the smallest is the + * local cell aspect ratio. That makes the aspect ratio available from the dual grid alone, which + * SetControlVolume builds on every multigrid level, whereas CGeometry::Aspect_Ratio exists only on + * MESH_0. The same quantity decides where LINELET preconditioner lines stop, in GetLineletInfo. + * + * Measuring the whole grid once keeps this out of the line-growth loop, which previously rescanned + * every neighbour of a node to find its weakest edge on every step of every line. ---*/ + const auto nPointFine = fine_grid->GetnPoint(); - const unsigned long nPointFine = fine_grid->GetnPoint(); - const unsigned long starting_Index_CoarseCV = Index_CoarseCV; - const bool DEBUG_OUTPUT = (rank == MASTER_NODE); + CNodeStiffness stiff; + stiff.wMin.assign(nPointFine, 0.0); + stiff.wMax.assign(nPointFine, 0.0); + stiff.jStiffest.assign(nPointFine, std::numeric_limits::max()); - /*--- How many parallel lines one coarse CV may span tangential to the wall. In 2D a wall "face" is - * an edge with 2 end nodes, in 3D a quadrilateral with 4 corner nodes, which is the number of - * lines that must be bundled to coarsen by 2 in every wall-tangential direction. ---*/ - unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; - if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; + for (auto iPoint = 0ul; iPoint < nPointFine; ++iPoint) { + su2double wmin = std::numeric_limits::max(), wmax = 0.0; + auto jStiffest = std::numeric_limits::max(); + + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); ++iNeigh) { + const auto jPoint = fine_grid->nodes->GetPoint(iPoint, iNeigh); + const auto iEdge = fine_grid->nodes->GetEdge(iPoint, iNeigh); + const su2double area = GeometryToolbox::Norm(nDim, fine_grid->edges->GetNormal(iEdge)); + const su2double w = + 0.5 * area * (1.0 / fine_grid->nodes->GetVolume(iPoint) + 1.0 / fine_grid->nodes->GetVolume(jPoint)); + if (w > wmax) { + wmax = w; + jStiffest = jPoint; + } + wmin = std::min(wmin, w); + } + + /*--- A node with no neighbours keeps the zeroed defaults, so AspectRatio reads 1. ---*/ + if (jStiffest != std::numeric_limits::max()) { + stiff.wMin[iPoint] = wmin; + stiff.wMax[iPoint] = wmax; + stiff.jStiffest[iPoint] = jStiffest; + } + } + return stiff; +} + +vector> CMultiGridGeometry::BuildImplicitLines(const CGeometry* fine_grid, const CConfig* config, + const CNodeStiffness& stiff) const { + /*--- Stop a line where its direction deviates by more than this from the previous step. ---*/ + constexpr su2double ANGLE_THRESHOLD_DEG = 20.0; + /*--- Fraction of a marker's nodes that must sit in a layer before the whole marker may seed. ---*/ + constexpr su2double QUALIFIED_FRACTION = 0.5; - /*--- Smallest local cell aspect ratio that still counts as a stretched layer. ---*/ + const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); + const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; const su2double MIN_AR = config->GetMGOptions().MG_Implicit_Lines_Min_AR; const bool USE_AR = (MIN_AR > 1.0); - /*--- Strength of the coupling across the dual face between a node and one of its neighbours. For a - * cell of streamwise size dx and wall-normal size dy this is 1/dy across the wall-normal face and - * 1/dx across the tangential one, so the ratio of the largest weight at a node to the smallest is - * the local cell aspect ratio. That makes the aspect ratio available from the dual grid alone, - * which SetControlVolume builds on every multigrid level, whereas CGeometry::Aspect_Ratio exists - * only on MESH_0. The same quantity already decides where LINELET preconditioner lines stop, in - * CGeometry::GetLineletInfo. ---*/ - auto edgeWeight = [&](unsigned long iPoint, unsigned short iNeigh) { - const auto jPoint = fine_grid->nodes->GetPoint(iPoint, iNeigh); - const auto iEdge = fine_grid->nodes->GetEdge(iPoint, iNeigh); - const su2double area = GeometryToolbox::Norm(nDim, fine_grid->edges->GetNormal(iEdge)); - return 0.5 * area * (1.0 / fine_grid->nodes->GetVolume(iPoint) + 1.0 / fine_grid->nodes->GetVolume(jPoint)); - }; + const auto nPointFine = fine_grid->GetnPoint(); + const auto nMarkerFine = fine_grid->GetnMarker(); + constexpr auto NO_POINT = std::numeric_limits::max(); - /*--- Weakest coupling at a node, i.e. the denominator of the local aspect ratio. ---*/ - auto minEdgeWeight = [&](unsigned long iPoint) { - su2double wmin = std::numeric_limits::max(); - for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) - wmin = std::min(wmin, edgeWeight(iPoint, iNeigh)); - return wmin; - }; + vector> lines; + vector> dir; /*!< Current marching direction of each line. */ + vector claimed(nPointFine, 0); - /*--- Aspect ratio of the mesh at iPoint measured along the edge to jPoint, and the neighbour the - * stiffest edge leads to. Taking the weight of one specific edge over the weakest edge at the - * node, rather than the largest over the smallest, keeps the measure directional: a mesh graded - * in the streamwise direction reads as stretched to the undirected form even far from any wall, - * which is why GetLineletInfo's min/max test cannot be used to decide where a line ends. ---*/ - auto aspectRatioAlong = [&](unsigned long iPoint, unsigned long jPoint) { - const su2double wmin = minEdgeWeight(iPoint); - if (wmin <= 0.0) return su2double(1.0); - for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) - if (fine_grid->nodes->GetPoint(iPoint, iNeigh) == jPoint) return edgeWeight(iPoint, iNeigh) / wmin; - return su2double(1.0); + auto isWall = [&](unsigned short bc) { + return (bc == HEAT_FLUX) || (bc == ISOTHERMAL) || (bc == CHT_WALL_INTERFACE) || (bc == SMOLUCHOWSKI_MAXWELL); }; - /*================================================================================================== - * PHASE A - build the implicit lines. - * - * Lines are grown one step at a time across ALL lines simultaneously rather than one line to - * completion at a time, and a node is claimed globally the moment any line takes it. Growing them - * one-at-a-time lets an early line run the full depth of the layer and consume nodes that a later, - * neighbouring line needed, so that later line terminates after a step or two; the lines then have - * wildly different lengths and cannot be bundled into columns of uniform depth. Advancing in - * lockstep makes all lines compete for each layer on equal terms, which on an extruded prismatic - * layer reproduces the mesh's own structure: every line reaches the same depth. - *================================================================================================*/ - vector> lines; /*!< lines[i] = [wall_node, interior_1, interior_2, ...] */ - vector dir; /*!< Current marching direction of each line, nDim per line. */ - vector claimed(nPointFine, 0); /*!< Node already belongs to some line. */ - - /*--- Nodes that sit on a boundary with a boundary condition on it. A line must not grow into one, - * because those nodes belong to the boundary agglomeration and a stack that absorbed one would - * straddle two boundaries. CPoint's Boundary flag cannot answer this on its own: it is set for - * every marker a node belongs to, SEND_RECEIVE included, so on a partitioned mesh it is also - * true for the ordinary interior nodes of the send fringe. Testing it directly would stop every - * line that reaches the fringe one layer short of the partition, leaving the top of those - * columns to isotropic agglomeration purely because of where the mesh was cut. ---*/ + /*--- Nodes on a boundary that carries a boundary condition. A line must not grow into one: those + * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two + * boundaries. CPoint's Boundary flag cannot answer this, as it is also set by SEND_RECEIVE, so on + * a partitioned mesh it is true for ordinary interior nodes of the send fringe and every line + * would stop one layer short of the partition. Walking the markers' own vertex lists is both + * exact and cheaper than testing every point against every marker. ---*/ vector onPhysicalBoundary(nPointFine, 0); - for (auto iPoint = 0ul; iPoint < nPointFine; ++iPoint) { - if (!fine_grid->nodes->GetBoundary(iPoint)) continue; - for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; - if (fine_grid->nodes->GetVertex(iPoint, iMarker) != -1) { - onPhysicalBoundary[iPoint] = 1; - break; - } - } + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) + onPhysicalBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } - /*--- Seed a line at iPoint growing away from the boundary along unitNormal. ---*/ - auto seedLine = [&](unsigned long iPoint, const su2double* unitNormal) { - lines.push_back({iPoint}); - claimed[iPoint] = 1; - for (unsigned short d = 0; d < nDim; ++d) dir.push_back(unitNormal[d]); - }; - - /*--- Unit normal of the boundary at a vertex, false if the marker does not reach iPoint. ---*/ + /*--- Unit normal of a boundary at a vertex, false if the marker does not reach iPoint. ---*/ auto vertexNormal = [&](unsigned long iPoint, unsigned short iMarker, su2double* unitNormal) { const long ChildVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); if (ChildVertex == -1) return false; @@ -1438,91 +1543,72 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, return true; }; - /*--- Viscous walls always carry a stretched layer, so they seed unconditionally. Running them - * first also settles the nodes where a wall meets another boundary: the wall claims them, and - * the line there is a wall line. ---*/ - for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - const auto bc = config->GetMarker_All_KindBC(iMarker); - if (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL) continue; + /*--- True if the mesh at iPoint is stretched along the boundary normal, i.e. this boundary has a + * layer growing off it the way a viscous wall does. On the side planes of a bump the mesh is just + * as stretched, but in a direction running ALONG the plane, and those nodes belong to the wall. ---*/ + auto hasLayerNormalTo = [&](unsigned long iPoint, const su2double* unitNormal) { + const auto jStiffest = stiff.jStiffest[iPoint]; + if (jStiffest == NO_POINT) return false; + if (stiff.AspectRatio(iPoint) < MIN_AR) return false; + + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jStiffest), fine_grid->nodes->GetCoord(iPoint), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) return false; + for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; + return fabs(GeometryToolbox::DotProduct(nDim, vec, unitNormal)) >= cos_threshold; + }; + /*--- Seed a line at every eligible node of a marker. ---*/ + auto seedMarker = [&](unsigned short iMarker, bool requireLayer) { for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); if (!fine_grid->nodes->GetDomain(iPoint)) continue; if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; - if (claimed[iPoint]) continue; /*--- A node on two wall markers must seed only one line. ---*/ + if (claimed[iPoint]) continue; /*--- A node on two markers must seed only one line. ---*/ su2double Normal[MAXNDIM] = {0.0}; if (!vertexNormal(iPoint, iMarker, Normal)) continue; - seedLine(iPoint, Normal); + if (requireLayer && !hasLayerNormalTo(iPoint, Normal)) continue; + + lines.push_back({iPoint}); + claimed[iPoint] = 1; + std::array d0{}; + for (unsigned short d = 0; d < nDim; ++d) d0[d] = Normal[d]; + dir.push_back(d0); } - } + }; - /*================================================================================================== - * Boundaries other than viscous walls that nevertheless carry a stretched layer normal to - * themselves. A symmetry plane laid in the same surface as a wall, such as the one ahead of a - * flat plate's leading edge or the flat floor sections either side of a bump, is meshed with the - * very same normal spacing as the wall it continues. Seeding only from walls leaves the mesh - * above it to isotropic agglomeration, so the coarse grid changes character across the line where - * the two meet even though the fine grid does not, and that shows up as a residual there. - * - * A node qualifies when its stiffest edge is both stretched and points along the boundary normal, - * which is what distinguishes a layer growing off this boundary from one merely passing by: on - * the side planes of a bump the mesh is just as stretched, but in the wall-normal direction that - * runs ALONG the plane, and those nodes belong to the wall's own lines. + /*--- Viscous walls always carry a stretched layer, so they seed unconditionally. Running them first + * also settles nodes where a wall meets another boundary: the wall claims them. ---*/ + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) + if (isWall(config->GetMarker_All_KindBC(iMarker))) seedMarker(iMarker, false); + + /*--- Boundaries other than viscous walls that nevertheless carry a stretched layer normal to + * themselves, such as a symmetry plane laid in the same surface as a wall - the floor either side + * of a bump, or ahead of a flat plate's leading edge. Seeding only from walls leaves the mesh + * above those to isotropic agglomeration, so the coarse grid changes character across a line the + * fine grid does not single out. * - * The decision is then taken per marker rather than per node. Seeding individual qualifying nodes - * on a marker that mostly does not qualify scatters isolated lines across a face whose neighbours - * seed nothing, and those become one-line bundles, i.e. coarse CVs one fine CV wide that do not - * coarsen tangentially at all. Measured on a flat plate and a 3D bump the two populations are far - * apart - boundaries with a layer normal to them qualify at 100%, while side planes, inlets, - * outlets and far fields come in at 13% and below - so any threshold near a half separates them. - *================================================================================================*/ + * The verdict is taken per marker, not per node: seeding isolated qualifying nodes on a marker + * that mostly does not qualify scatters one-line bundles, i.e. coarse CVs that do not coarsen + * tangentially at all. The two populations are far apart in practice - boundaries with a layer + * normal to them qualify at 100%, side planes, inlets and far fields at 13% and below - so any + * threshold near a half separates them. ---*/ if (USE_AR) { - const auto nMarkerFine = fine_grid->GetnMarker(); - - /*--- Counts are kept per marker of the configuration file, not per local marker. Ranks do not - * agree on either the number of markers or their order, because the SEND_RECEIVE markers of a - * partition are appended to its own list, so the same index means a different boundary on - * another rank and a reduction over it would add unrelated boundaries together. The - * configuration file list is the same everywhere. ---*/ + /*--- Counted per configuration-file marker, not per local marker: ranks agree on neither the + * number nor the order of local markers, because each partition appends its own SEND_RECEIVE + * markers, so the same index means a different boundary elsewhere. ---*/ const auto nMarkerCfg = config->GetnMarker_CfgFile(); vector nValid(nMarkerCfg, 0), nQualified(nMarkerCfg, 0); - /*--- True if the mesh at iPoint is stretched along the boundary normal, i.e. this boundary has a - * layer growing off it in the same way a viscous wall does. ---*/ - auto hasLayerNormalTo = [&](unsigned long iPoint, const su2double* unitNormal) { - su2double wmax = 0.0, wmin = std::numeric_limits::max(); - unsigned long jStiffest = ULONG_MAX; - for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(iPoint); iNeigh++) { - const su2double w = edgeWeight(iPoint, iNeigh); - if (w > wmax) { - wmax = w; - jStiffest = fine_grid->nodes->GetPoint(iPoint, iNeigh); - } - wmin = std::min(wmin, w); - } - if (jStiffest == ULONG_MAX || wmin <= 0.0) return false; - if (wmax / wmin < MIN_AR) return false; - - su2double vec[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jStiffest), fine_grid->nodes->GetCoord(iPoint), vec); - const su2double len = GeometryToolbox::Norm(nDim, vec); - if (len <= 0.0) return false; - for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - return fabs(GeometryToolbox::DotProduct(nDim, vec, unitNormal)) >= cos_threshold; - }; - - /*--- Markers that may be tested at all. Periodic boundaries are left out: the two halves are the - * same physical location under a transform and have their own matching, which a line running - * into one would interfere with. ---*/ auto canSeed = [&](unsigned short iMarker) { const auto bc = config->GetMarker_All_KindBC(iMarker); - if (bc == SEND_RECEIVE || bc == PERIODIC_BOUNDARY) return false; - return (bc != HEAT_FLUX && bc != ISOTHERMAL && bc != CHT_WALL_INTERFACE && bc != SMOLUCHOWSKI_MAXWELL); + /*--- Periodic boundaries are left out: the two halves are the same physical location under a + * transform and have their own matching, which a line running into one would disturb. ---*/ + return (bc != SEND_RECEIVE) && (bc != PERIODIC_BOUNDARY) && !isWall(bc); }; - /*--- Position of a local marker in the configuration file list. Only meaningful for the markers - * canSeed accepts: a SEND_RECEIVE marker is named per partition and is not in that list. ---*/ vector cfgOfMarker(nMarkerFine, 0); for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) if (canSeed(iMarker)) @@ -1540,8 +1626,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } - /*--- A marker is generally split over several ranks, so the verdict has to be taken on the whole - * of it or two ranks could disagree about the same boundary. ---*/ + /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. ---*/ if (nMarkerCfg > 0) { vector tmp(nMarkerCfg); SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -1554,43 +1639,38 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (!canSeed(iMarker)) continue; const auto iCfg = cfgOfMarker[iMarker]; if (nValid[iCfg] == 0) continue; - if (2 * nQualified[iCfg] < nValid[iCfg]) continue; /*--- Fewer than half, not a layer. ---*/ - - for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { - const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); - if (!fine_grid->nodes->GetDomain(iPoint)) continue; - if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; - if (claimed[iPoint]) continue; - - su2double Normal[MAXNDIM] = {0.0}; - if (!vertexNormal(iPoint, iMarker, Normal)) continue; - /*--- The marker carries a layer, but this node still has to be in it. ---*/ - if (!hasLayerNormalTo(iPoint, Normal)) continue; - seedLine(iPoint, Normal); - } + if (su2double(nQualified[iCfg]) < QUALIFIED_FRACTION * su2double(nValid[iCfg])) continue; + seedMarker(iMarker, true); } } - if (lines.empty()) return; - + /*--- Grow every line one step per sweep rather than one line to completion at a time, claiming a + * node globally the moment any line takes it. Growing them one at a time lets an early line run + * the full depth of the layer and consume nodes a neighbouring line needed, so that line stops + * after a step or two and the lengths become too uneven to bundle into columns of uniform depth. + * In lockstep all lines compete for each layer on equal terms, which on an extruded prismatic + * layer reproduces the mesh's own structure. ---*/ vector growing(lines.size(), 1); + for (bool any_grew = true; any_grew;) { any_grew = false; + for (unsigned long li = 0; li < lines.size(); ++li) { if (!growing[li]) continue; if (lines[li].size() >= MAX_LINE_LENGTH) { growing[li] = 0; continue; } + const auto current = lines[li].back(); - su2double best_dot = -2.0; - unsigned long best_neighbor = ULONG_MAX; + su2double best_dot = -2.0, best_dir[MAXNDIM] = {0.0}; + auto best = NO_POINT; + unsigned short best_neigh = 0; - for (auto jPoint : fine_grid->nodes->GetPoints(current)) { + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(current); ++iNeigh) { + const auto jPoint = fine_grid->nodes->GetPoint(current, iNeigh); /*--- Halo nodes stay out: their parent is dictated by the rank that owns them and arrives - * through the MPI relay, so a line claiming one here would fight that assignment. A line - * therefore still ends at the partition itself, but now only there, instead of one layer - * earlier at the fringe of owned nodes. ---*/ + * through the MPI relay, so a line claiming one would fight that assignment. ---*/ if (!fine_grid->nodes->GetDomain(jPoint)) continue; if (onPhysicalBoundary[jPoint]) continue; if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; @@ -1602,187 +1682,249 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (len <= 0.0) continue; for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - const su2double dot = GeometryToolbox::DotProduct(nDim, vec, &dir[li * nDim]); + const su2double dot = GeometryToolbox::DotProduct(nDim, vec, dir[li].data()); if (dot > best_dot) { best_dot = dot; - best_neighbor = jPoint; + best = jPoint; + best_neigh = iNeigh; + for (unsigned short d = 0; d < nDim; ++d) best_dir[d] = vec[d]; } } - if (best_neighbor == ULONG_MAX || best_dot < cos_threshold) { + if ((best == NO_POINT) || (best_dot < cos_threshold)) { growing[li] = 0; continue; } /*--- End the line where the mesh stops being stretched along it. Without this the only limits * are the direction cone and MAX_LINE_LENGTH, so a line leaves the boundary layer and keeps - * going into the far field, stacking coarse CVs along a direction the fine grid does not - * single out. Ordinary agglomeration handles that region better. ---*/ - if (USE_AR && aspectRatioAlong(current, best_neighbor) < MIN_AR) { - growing[li] = 0; - continue; - } - - su2double step[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(best_neighbor), fine_grid->nodes->GetCoord(current), - step); - const su2double slen = GeometryToolbox::Norm(nDim, step); - if (slen <= 0.0) { - growing[li] = 0; - continue; + * stacking coarse CVs along a direction the fine grid does not single out. Taking the weight + * of this one edge over the weakest edge at the node keeps the measure directional: a mesh + * graded in the streamwise direction reads as stretched to an undirected min/max test even + * far from any wall. ---*/ + if (USE_AR && (stiff.wMin[current] > 0.0)) { + const auto jPoint = fine_grid->nodes->GetPoint(current, best_neigh); + const auto iEdge = fine_grid->nodes->GetEdge(current, best_neigh); + const su2double area = GeometryToolbox::Norm(nDim, fine_grid->edges->GetNormal(iEdge)); + const su2double w = + 0.5 * area * (1.0 / fine_grid->nodes->GetVolume(current) + 1.0 / fine_grid->nodes->GetVolume(jPoint)); + if (w / stiff.wMin[current] < MIN_AR) { + growing[li] = 0; + continue; + } } - for (unsigned short d = 0; d < nDim; ++d) dir[li * nDim + d] = step[d] / slen; - lines[li].push_back(best_neighbor); - claimed[best_neighbor] = 1; + for (unsigned short d = 0; d < nDim; ++d) dir[li][d] = best_dir[d]; + lines[li].push_back(best); + claimed[best] = 1; any_grew = true; } } - /*--- A line needs at least one interior node to contribute anything. Drop the rest and release - * their wall seed so ordinary boundary agglomeration can treat it normally. ---*/ - { - vector> kept; - kept.reserve(lines.size()); - for (auto& L : lines) { - if (L.size() >= 2) - kept.push_back(std::move(L)); - else - claimed[L[0]] = 0; - } - lines = std::move(kept); - } - if (lines.empty()) return; + /*--- A line needs at least one interior node to contribute anything. ---*/ + vector> kept; + kept.reserve(lines.size()); + for (auto& L : lines) + if (L.size() >= 2) kept.push_back(std::move(L)); - /*================================================================================================== - * PHASE B - partition the lines into bundles. - * - * Every line must end up in exactly one bundle, and a bundle must be a compact patch on the wall: - * in 3D the four lines rising from the corners of one wall quadrilateral, in 2D the two lines from - * the ends of one wall edge. Selecting, for each line independently, some set of neighbours to - * merge with does not do this - the relation is not symmetric, so line 1 claiming {2,3,4} does not - * stop line 2 from claiming {1,3,5}, and the bundles overlap and fight over nodes. - * - * Building the partition by repeated pairwise matching avoids that by construction. One matching - * round pairs adjacent lines into 2-bundles (the wall edge); a second round pairs adjacent - * 2-bundles into 4-bundles (the wall quadrilateral). Each round is a matching, so membership is - * mutually exclusive at every stage and the result is a true partition. It also needs nothing but - * point-to-point connectivity, so it works identically on every multigrid level - boundary face - * connectivity does not exist on agglomerated grids, so a literal "same quadrilateral" test would - * only ever work for the first coarsening. + return kept; +} + +vector> CMultiGridGeometry::BundleImplicitLines(const vector>& lines, + const CGeometry* fine_grid, + const CConfig* config, + vector>& adj) const { + /*--- Every line must end up in exactly one bundle, and a bundle must be a compact patch on the wall: + * in 3D the four lines rising from the corners of one wall quadrilateral, in 2D the two lines from + * the ends of one wall edge. Choosing, for each line independently, a set of neighbours to merge + * with does not do this - the relation is not symmetric, so line 1 claiming {2,3} does not stop + * line 2 claiming {1,4}, and the bundles overlap and fight over nodes. * - * Lines may only be bundled when their wall nodes carry the same set of physical markers, so that - * a bundle never straddles a boundary-condition change (the same rule ordinary agglomeration uses: - * ridges merge only with ridges, valleys only with valleys). - *================================================================================================*/ - const unsigned long nLines = lines.size(); + * Repeated pairwise matching avoids that by construction. One round pairs adjacent lines into the + * wall edge, a second pairs adjacent pairs into the wall quadrilateral. Each round is a matching, + * so membership stays mutually exclusive and the result is a true partition. It needs nothing but + * point-to-point connectivity, so it works identically on every multigrid level - boundary face + * connectivity does not survive agglomeration, so a literal "same quadrilateral" test would only + * ever work for the first coarsening. Two rounds reach 4, which is the 3D group size, so the + * number of rounds follows from max_group instead of iterating to a fixed point. ---*/ + const auto nLines = lines.size(); + unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; + if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; - /*--- Marker signature of each line's wall node. ---*/ - vector> sig(nLines); - for (unsigned long li = 0; li < nLines; ++li) { - for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; - if (fine_grid->nodes->GetVertex(lines[li][0], iMarker) != -1) sig[li].push_back(iMarker); + /*--- Marker signature of each line's boundary node, as a bitmask over the physical markers. Lines + * may only be bundled when these match, so a bundle never straddles a change of boundary + * condition - the rule ordinary agglomeration uses for ridges and valleys. ---*/ + const auto nMarkerFine = fine_grid->GetnMarker(); + vector physBit(nMarkerFine, -1); + unsigned nPhys = 0; + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) + if (config->GetMarker_All_KindBC(iMarker) != SEND_RECEIVE) physBit[iMarker] = static_cast(nPhys++); + + const unsigned nWords = std::max(1u, (nPhys + 63u) / 64u); + vector sig(nLines * nWords, 0); + for (unsigned long li = 0; li < nLines; ++li) + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (physBit[iMarker] < 0) continue; + if (fine_grid->nodes->GetVertex(lines[li][0], iMarker) == -1) continue; + const auto b = static_cast(physBit[iMarker]); + sig[li * nWords + b / 64] |= (uint64_t(1) << (b % 64)); } - sort(sig[li].begin(), sig[li].end()); - } + auto sameSig = [&](unsigned long la, unsigned long lb) { + for (unsigned w = 0; w < nWords; ++w) + if (sig[la * nWords + w] != sig[lb * nWords + w]) return false; + return true; + }; - /*--- Line adjacency, inherited from the wall nodes' mesh connectivity. ---*/ - unordered_map lineOfWallNode; - lineOfWallNode.reserve(nLines); - for (unsigned long li = 0; li < nLines; ++li) lineOfWallNode[lines[li][0]] = li; + /*--- Line adjacency, inherited from the boundary nodes' mesh connectivity. ---*/ + vector lineOfWallNode(fine_grid->GetnPoint(), -1); + for (unsigned long li = 0; li < nLines; ++li) lineOfWallNode[lines[li][0]] = static_cast(li); - vector> adj(nLines); - for (unsigned long li = 0; li < nLines; ++li) { + adj.assign(nLines, {}); + for (unsigned long li = 0; li < nLines; ++li) for (auto jPoint : fine_grid->nodes->GetPoints(lines[li][0])) { - const auto it = lineOfWallNode.find(jPoint); - if (it != lineOfWallNode.end() && it->second != li) adj[li].push_back(it->second); + const auto lj = lineOfWallNode[jPoint]; + if ((lj >= 0) && (static_cast(lj) != li)) adj[li].push_back(static_cast(lj)); } - } - /*--- Round 1: match adjacent lines into pairs. ---*/ - vector> bundles; - vector bundleOf(nLines, -1); - bundles.reserve(nLines); + vector> groups(nLines); + vector groupOf(nLines); for (unsigned long li = 0; li < nLines; ++li) { - if (bundleOf[li] >= 0) continue; - const long b = static_cast(bundles.size()); - bundles.push_back({li}); - bundleOf[li] = b; - if (max_group < 2) continue; - for (auto lj : adj[li]) { - if (bundleOf[lj] >= 0 || sig[lj] != sig[li]) continue; - bundles[b].push_back(lj); - bundleOf[lj] = b; - break; - } + groups[li] = {li}; + groupOf[li] = li; } - /*--- Further rounds: merge adjacent bundles while they still fit. In 3D this turns pairs into - * quadrilaterals; in 2D max_group is 2 so it only absorbs leftover singletons. ---*/ - for (bool changed = true; changed;) { - changed = false; - - vector> badj(bundles.size()); - for (unsigned long li = 0; li < nLines; ++li) - for (auto lj : adj[li]) - if (bundleOf[li] != bundleOf[lj]) badj[bundleOf[li]].push_back(bundleOf[lj]); - for (auto& v : badj) { - sort(v.begin(), v.end()); - v.erase(unique(v.begin(), v.end()), v.end()); - } + const unsigned nRounds = (max_group <= 1) ? 0 : ((max_group <= 2) ? 1 : 2); + vector> shared; - vector consumed(bundles.size(), 0); + for (unsigned round = 0; round < nRounds; ++round) { + vector consumed(groups.size(), 0); vector> merged; - merged.reserve(bundles.size()); - for (unsigned long b = 0; b < bundles.size(); ++b) { - if (consumed[b]) continue; - consumed[b] = 1; - auto group = bundles[b]; - for (auto h : badj[b]) { - if (consumed[h]) continue; - if (group.size() + bundles[h].size() > max_group) continue; - if (sig[bundles[h].front()] != sig[group.front()]) continue; - consumed[h] = 1; - group.insert(group.end(), bundles[h].begin(), bundles[h].end()); - changed = true; - break; + merged.reserve(groups.size()); + + for (unsigned long g = 0; g < groups.size(); ++g) { + if (consumed[g]) continue; + consumed[g] = 1; + auto group = groups[g]; + + /*--- Count how many line-to-line adjacencies this group shares with each candidate. Merging the + * candidate that shares the most keeps the patch square: a pair lying alongside this one + * touches it along its whole length and shares two adjacencies, whereas a pair continuing in + * the same direction touches at one end and shares one. Taking the first candidate that fits + * instead, as this did, produces a 1x4 strip of lines about as often as the mesh offers one, + * and those extrude into coarse CVs elongated in one wall-tangential direction. ---*/ + shared.clear(); + for (auto li : group) + for (auto lj : adj[li]) { + const auto h = groupOf[lj]; + if ((h == g) || consumed[h]) continue; + if (group.size() + groups[h].size() > max_group) continue; + if (!sameSig(groups[h].front(), group.front())) continue; + + bool seen = false; + for (auto& s : shared) + if (s.first == h) { + s.second++; + seen = true; + break; + } + if (!seen) shared.emplace_back(h, 1); + } + + auto bestH = std::numeric_limits::max(); + unsigned long bestShared = 0; + for (const auto& s : shared) + if (s.second > bestShared) { + bestShared = s.second; + bestH = s.first; + } + + if (bestH != std::numeric_limits::max()) { + consumed[bestH] = 1; + group.insert(group.end(), groups[bestH].begin(), groups[bestH].end()); } merged.push_back(std::move(group)); } - bundles = std::move(merged); - for (unsigned long b = 0; b < bundles.size(); ++b) - for (auto li : bundles[b]) bundleOf[li] = static_cast(b); + groups = std::move(merged); + for (unsigned long g = 0; g < groups.size(); ++g) + for (auto li : groups[g]) groupOf[li] = g; } + return groups; +} + +void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, + const CConfig* config) { + const auto starting_Index_CoarseCV = Index_CoarseCV; + + const auto stiff = ComputeNodeStiffness(fine_grid); + + /*--- PHASE A and B. Neither may be skipped on a rank with no lines: PHASE A takes a collective to + * agree on which markers carry a layer, and the summary at the end takes another. ---*/ + const auto lines = BuildImplicitLines(fine_grid, config, stiff); + + vector> adj, bundles; + if (!lines.empty()) bundles = BundleImplicitLines(lines, fine_grid, config, adj); + /*================================================================================================== * PHASE C - extrude each bundle into a stack of coarse control volumes. * - * The bundle's wall nodes become one coarse CV, and each successive layer of the bundle's lines - * becomes the next, so the coarse grid inherits the layer structure of the fine grid and a line - * relaxation remains meaningful on it. Because Phase A made the lines node-disjoint and Phase B - * made the bundles a partition, no two bundles can ever contend for the same node, so a stack is - * never interrupted part way up. + * The bundle's boundary nodes become one coarse CV and each successive layer becomes the next, so + * the coarse grid inherits the layer structure of the fine grid and a line relaxation stays + * meaningful on it. PHASE A made the lines node-disjoint and PHASE B made the bundles a partition, + * so no two bundles can contend for a node and a stack is never interrupted part way up. * - * The lines in a bundle need not be equally long. A stack therefore keeps rising for as long as - * enough of its lines have nodes left, narrowing where the shorter ones end, instead of stopping - * where the shortest one does and abandoning everything the taller ones still had. + * How many fine layers go into one coarse CV decides what is coarsened. One layer coarsens only + * tangentially and leaves the wall-normal line intact, which is what a line-implicit smoother needs + * where the cells are stretched; two layers coarsen in every direction, which is what the far field + * wants. MG_IMPLICIT_LINES_ISO_AR switches between them by the local aspect ratio as the stack + * rises, so the same stack can start semi-coarsened at the wall and finish isotropic - and because + * each semi-coarsening halves the stretching, the switch happens by itself at whatever level the + * mesh stops being anisotropic, rather than being tied to a multigrid level. With it at 0 the old + * behaviour stands and MG_IMPLICIT_LINES_ISOTROPIC decides for the whole stack. * - * The multigrid queue is deliberately not updated here: the sync loop that follows the boundary - * agglomeration removes every point already marked agglomerated, so removing them a second time - * from this function would be an error. + * The multigrid queue is deliberately not touched here: the sync loop after the boundary + * agglomeration removes every point already marked agglomerated, so doing it here too would be an + * error. *================================================================================================*/ - const unsigned long nBlock = ISOTROPIC ? 2 : 1; + const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; + const su2double ISO_AR = config->GetMGOptions().MG_Implicit_Lines_Iso_AR; + const bool HYBRID = (ISO_AR > 0.0); + + /*--- Are the still-growing lines one connected patch on the wall? A set that is no longer connected + * would put two separated columns into a single CV. Scans PHASE B's adjacency, over at most + * max_group entries, so the quadratic form is cheap. ---*/ + auto isConnected = [&adj](const vector& members, const vector& act) { + if (act.size() <= 1) return true; + vector seen(act.size(), 0); + vector stack{0}; + seen[0] = 1; + unsigned long nSeen = 1; + while (!stack.empty()) { + const auto cur = stack.back(); + stack.pop_back(); + const auto& neighbors = adj[members[act[cur]]]; + for (unsigned long k = 0; k < act.size(); ++k) { + if (seen[k]) continue; + if (find(neighbors.begin(), neighbors.end(), members[act[k]]) != neighbors.end()) { + seen[k] = 1; + nSeen++; + stack.push_back(k); + } + } + } + return nSeen == act.size(); + }; - map bundle_size_histogram; - unsigned long nStacks = 0, nTruncated = 0; + unsigned long nStacks = 0, nTruncated = 0, nSemiCV = 0, nFullCV = 0; + unsigned long histogram[9] = {0}; + vector placed, active, group; for (const auto& members : bundles) { - bundle_size_histogram[members.size()]++; + histogram[std::min(members.size(), 8)]++; - /*--- The wall CV. Claiming it here, before ordinary boundary agglomeration runs, is what keeps - * the whole stack aligned: the layer above a wall CV has exactly the same footprint. ---*/ + /*--- The boundary CV. Claiming it before ordinary boundary agglomeration runs is what keeps the + * stack aligned: the layer above has exactly the same footprint. ---*/ bool valid = true; for (auto li : members) if (!GeometricalCheck(lines[li][0], fine_grid, config)) valid = false; @@ -1797,55 +1939,41 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, Index_CoarseCV++; nStacks++; - /*--- The interior layers, in lockstep. A line that runs out simply stops contributing and the - * others carry on without it, so a bundle is no longer cut down to its shortest member: the - * stack narrows as it rises instead of ending. The CVs still form one connected column, which - * is what a line relaxation on the coarse grid needs. - * - * Two situations end the stack rather than narrowing it further. Dropping below two lines - * would extrude a column one line wide, thinner than anything the domain pass would build - * there, and a set of lines that is no longer connected on the wall would put two separated - * columns into a single CV. In both cases the nodes above are better left to ordinary domain - * agglomeration. A bundle that only ever had one line is exempt from the first rule: it is - * one line wide by construction, and stopping early would gain nothing. ---*/ - - /*--- Are the still-growing lines one connected patch on the wall? Takes positions into members, - * which is at most max_group long, so the quadratic scan over Phase B's adjacency is cheap. ---*/ - auto isConnected = [&adj, &members](const vector& act) { - if (act.size() <= 1) return true; - vector seen(act.size(), 0); - vector stack{0}; - seen[0] = 1; - unsigned long nSeen = 1; - while (!stack.empty()) { - const auto cur = stack.back(); - stack.pop_back(); - const auto& neighbors = adj[members[act[cur]]]; - for (unsigned long k = 0; k < act.size(); ++k) { - if (seen[k]) continue; - if (find(neighbors.begin(), neighbors.end(), members[act[k]]) != neighbors.end()) { - seen[k] = 1; - nSeen++; - stack.push_back(k); - } - } - } - return nSeen == act.size(); - }; - + /*--- The interior layers, in lockstep. A line that runs out stops contributing and the others + * carry on, so the stack narrows as it rises instead of being cut to its shortest member. + * Dropping below two lines would extrude a column one line wide, thinner than anything the + * domain pass would build there, so the stack ends instead; a bundle that only ever had one + * line is exempt, being one line wide by construction. ---*/ const unsigned long minActive = std::min(2, members.size()); + placed.assign(members.size(), 1); - vector placed(members.size(), 1); /*!< First node of each line not yet in a CV. */ - vector active, group; - - for (unsigned long first = 1;; first += nBlock) { - /*--- The lines that still have a whole block left at this height. ---*/ + auto collectActive = [&](unsigned long first, unsigned long blk) { active.clear(); for (unsigned long m = 0; m < members.size(); ++m) - if (first + nBlock <= lines[members[m]].size()) active.push_back(m); + if (first + blk <= lines[members[m]].size()) active.push_back(m); + }; + + for (unsigned long first = 1;;) { + unsigned long nBlock = ISOTROPIC ? 2 : 1; + if (HYBRID) { + /*--- Stay semi-coarsened while any line of this bundle is still in stretched mesh at this + * height; the wall-normal direction is shared by the whole stack. ---*/ + su2double arHere = 0.0; + for (unsigned long m = 0; m < members.size(); ++m) + if (first < lines[members[m]].size()) + arHere = std::max(arHere, stiff.AspectRatio(lines[members[m]][first])); + nBlock = (arHere > ISO_AR) ? 1 : 2; + } + + collectActive(first, nBlock); + /*--- One layer short of a full block at the top, take it as a single layer rather than drop it. ---*/ + if ((nBlock == 2) && (active.size() < minActive)) { + nBlock = 1; + collectActive(first, nBlock); + } if (active.size() < minActive) break; - if (!isConnected(active)) break; + if (!isConnected(members, active)) break; group.clear(); group.reserve(active.size() * nBlock); @@ -1863,21 +1991,41 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } nodes->SetnChildren_CV(Index_CoarseCV, static_cast(group.size())); Index_CoarseCV++; + ((nBlock == 1) ? nSemiCV : nFullCV)++; + for (auto m : active) placed[m] = first + nBlock; + first += nBlock; } - /*--- Whatever each line still carries above the last CV it contributed to. ---*/ for (unsigned long m = 0; m < members.size(); ++m) nTruncated += lines[members[m]].size() - placed[m]; } - if (DEBUG_OUTPUT) { - unsigned long nLineNodes = 0; - for (const auto& L : lines) nLineNodes += L.size(); - cout << " Implicit lines: " << nLines << " lines, " << nStacks << " stacks, bundle sizes "; - for (const auto& h : bundle_size_histogram) cout << h.first << "x" << h.second << " "; - cout << "\n Coarse CVs from lines: " << (Index_CoarseCV - starting_Index_CoarseCV) << " covering " - << (nLineNodes - nTruncated) << "/" << nLineNodes << " line nodes"; - if (nTruncated > 0) cout << " (" << nTruncated << " left to domain agglomeration)"; + /*--- Summary over all ranks. Reporting rank 0's own lines, as this used to, makes a partitioned run + * look like a fraction of the mesh it is not, and hides how much of the layer the partitioning + * cost: a line stops at the partition, so the count of nodes left to ordinary agglomeration is + * the number to watch when adding ranks. Every rank must reach these collectives. ---*/ + unsigned long nLineNodes = 0; + for (const auto& L : lines) nLineNodes += L.size(); + + unsigned long local[6] = {lines.size(), nStacks, nLineNodes, nTruncated, nSemiCV, nFullCV}; + unsigned long total[6] = {0}; + SU2_MPI::Allreduce(local, total, 6, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + + unsigned long localCV = Index_CoarseCV - starting_Index_CoarseCV, totalCV = 0; + SU2_MPI::Allreduce(&localCV, &totalCV, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + + unsigned long histTotal[9] = {0}; + SU2_MPI::Allreduce(histogram, histTotal, 9, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + + if (rank == MASTER_NODE) { + cout << " Implicit lines: " << total[0] << " lines, " << total[1] << " stacks, bundle sizes "; + for (unsigned s = 1; s <= 8; ++s) + if (histTotal[s] > 0) cout << s << "x" << histTotal[s] << " "; + cout << "\n Coarse CVs from lines: " << totalCV << " covering " << (total[2] - total[3]) << "/" << total[2] + << " line nodes"; + if (total[3] > 0) cout << " (" << total[3] << " left to domain agglomeration)"; + if (total[4] + total[5] > 0) + cout << "\n Stack layers: " << total[4] << " semi-coarsened, " << total[5] << " isotropic"; cout << endl; } } From 5b68d2e75f741bd6e796bfbd036bae06e504f08b Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 2 Sep 2026 22:11:26 +0200 Subject: [PATCH 37/54] advancing front implicit line method --- Common/include/CConfig.hpp | 6 + Common/include/geometry/CGeometry.hpp | 6 + .../include/geometry/CMultiGridGeometry.hpp | 77 +- Common/include/option_structure.hpp | 23 +- Common/src/CConfig.cpp | 40 +- Common/src/geometry/CGeometry.cpp | 3 +- Common/src/geometry/CMultiGridGeometry.cpp | 1430 +++++++++++++---- Common/src/geometry/CPhysicalGeometry.cpp | 451 +++++- Common/src/linear_algebra/CSysMatrix.cpp | 7 + config_template.cfg | 25 +- 10 files changed, 1641 insertions(+), 427 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 934f88feb22..dc218103e6e 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -1100,6 +1100,7 @@ class CConfig { long ParMETIS_pointWgt; /*!< \brief Load balancing weight given to points. */ long ParMETIS_edgeWgt; /*!< \brief Load balancing weight given to edges. */ su2double ParMETIS_anisoWgt; /*!< \brief Strength of the anisotropy-aware ParMETIS edge weights. 0 disables them. */ + bool ParMETIS_columnPart; /*!< \brief Partition contracted wall-normal columns instead of individual points. */ unsigned short DirectDiff; /*!< \brief Direct Differentation mode. */ bool DiscreteAdjoint, /*!< \brief AD-based discrete adjoint mode. */ DiscreteAdjointDebug; /*!< \brief Discrete adjoint debug mode using tags. */ @@ -10179,6 +10180,11 @@ class CConfig { */ passivedouble GetParMETIS_AnisoWeight() const { return SU2_TYPE::GetValue(ParMETIS_anisoWgt); } + /*! + * \brief Partition contracted wall-normal columns rather than individual points. + */ + bool GetParMETIS_ColumnPartition() const { return ParMETIS_columnPart; } + /*! * \brief Find the marker index (if any) that is part of a given interface pair. * \param[in] iInterface - Number of the interface pair being tested, starting at 0. diff --git a/Common/include/geometry/CGeometry.hpp b/Common/include/geometry/CGeometry.hpp index 9d3e574b432..54d0d76c0ec 100644 --- a/Common/include/geometry/CGeometry.hpp +++ b/Common/include/geometry/CGeometry.hpp @@ -228,6 +228,12 @@ class CGeometry { std::vector> linelets; /*!< \brief Point indices for each linelet. */ + /*!< \brief Whether the structure has been built. Not the same as having any linelets: a rank whose + * part of the mesh holds no solid wall builds an empty set, and inferring "not built yet" from + * that emptiness makes it rebuild - and re-run the collectives at the end of the construction - + * on every call, while every other rank answers from its cache and never calls them again. */ + bool built = false; + /*!< \brief Index of the linelet of each point ("linelets" transfered to points). */ std::vector lineletIdx; diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index 2dd83b0fd0e..601fb891d69 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -48,7 +48,20 @@ class CMultiGridGeometry final : public CGeometry { * \return TRUE or FALSE depending if the control volume can be agglomerated. */ bool SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, const CGeometry* fine_grid, - const CConfig* config) const; + const CConfig* config, const vector& mixedBC) const; + + /*! + * \brief Nodes carrying two or more physical boundary conditions of DIFFERENT type, e.g. the point + * where a wall ends against an outlet. Nishikawa's rules never agglomerate these: merging one + * into a coarse control volume averages two conditions that the fine grid applies separately, + * and neither ends up applied where it belongs. Two markers of the SAME type meeting - two + * wall patches, say - are not affected, nor is a node whose only second marker is + * SEND_RECEIVE, which records a partition and not a boundary condition. + * \param[in] fine_grid - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \return One flag per fine grid point, set where that point must stay on its own. + */ + vector FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const; /*! * \brief Determine if a Point can be agglomerated using geometrical criteria. @@ -91,7 +104,7 @@ class CMultiGridGeometry final : public CGeometry { * the local cell aspect ratio, available on every multigrid level unlike CGeometry::Aspect_Ratio. */ struct CNodeStiffness { - vector wMin, wMax; /*!< \brief Weakest and strongest edge coupling at each node. */ + vector wMin, wMax; /*!< \brief Weakest and strongest edge coupling at each node. */ vector jStiffest; /*!< \brief Neighbour across the strongest edge. */ /*!< \brief Local aspect ratio at a node, 1.0 where it could not be measured. */ @@ -108,28 +121,58 @@ class CMultiGridGeometry final : public CGeometry { CNodeStiffness ComputeNodeStiffness(const CGeometry* fine_grid) const; /*! - * \brief PHASE A of the implicit-line agglomeration: grow wall-normal lines of nodes from the - * boundaries that carry a stretched layer. Lines are node-disjoint. + * \brief Diagnostic tally of why each front stopped advancing, indexed by the STOP_* constants + * below. + * + * A front is stopped by exactly two things: reaching a boundary, or failing to lay a next layer + * that is topologically identical to the one it is standing on. Every reason below is one of those + * two. There is deliberately no criterion on direction or on how stretched the mesh is - a front + * runs until the mesh itself stops offering a clean extrusion. + */ + enum { + STOP_PHYS_BOUNDARY = 0, /*!< \brief Reached a boundary. The one expected, correct stop. */ + STOP_PARTITION, /*!< \brief Reached a partition interface, where the layer cannot be claimed. */ + STOP_COLLISION, /*!< \brief Lost a candidate to another front, i.e. the fronts met. */ + STOP_PINCH, /*!< \brief Two nodes of this front wanted the same successor. */ + STOP_AGGLOMERATED, /*!< \brief Ran into nodes an earlier phase had already taken. */ + STOP_NO_NEIGHBOR, /*!< \brief A front node had no free neighbour left to step onto. */ + STOP_TOPOLOGY, /*!< \brief The next layer was not isomorphic to the current one. */ + STOP_GEOMETRY, /*!< \brief A node of the next layer failed GeometricalCheck. */ + STOP_MAX_LENGTH, /*!< \brief Hit the MG_IMPLICIT_LINES_MAX_LENGTH safety cap, off by default. */ + N_STOP_REASONS + }; + + /*! + * \brief Boundary nodes that seed an advancing front, with the direction each starts marching in. + */ + struct CFrontSeeds { + vector node; /*!< \brief Seed node on the boundary. */ + vector> normal; /*!< \brief Unit normal there, pointing into the domain. */ + }; + + /*! + * \brief PHASE 1a of the paving agglomeration: collect the boundary nodes that seed an advancing + * front, i.e. those on a viscous wall, or on another boundary that carries a stretched + * layer normal to itself. * \param[in] fine_grid - Fine grid geometry. - * \param[in] config - Configuration. + * \param[in] config - Definition of the particular problem. * \param[in] stiff - Node coupling from ComputeNodeStiffness. - * \return One vector per line, each starting at its boundary node. + * \return Seed nodes and their inward boundary normals. */ - vector> BuildImplicitLines(const CGeometry* fine_grid, const CConfig* config, - const CNodeStiffness& stiff) const; + CFrontSeeds SeedFrontNodes(const CGeometry* fine_grid, const CConfig* config, const CNodeStiffness& stiff) const; /*! - * \brief PHASE B of the implicit-line agglomeration: partition the lines into compact bundles that - * share a wall footprint, by repeated pairwise matching. - * \param[in] lines - Lines from BuildImplicitLines. + * \brief PHASE 1b of the paving agglomeration: partition the seed nodes into compact surface + * patches by repeated pairwise matching. Each patch is the footprint of one front, and is + * the only thing that decides the shape of the whole stack above it. + * \param[in] seeds - Seed nodes from SeedFrontNodes. * \param[in] fine_grid - Fine grid geometry. - * \param[in] config - Configuration. - * \param[out] adj - Line adjacency inherited from the boundary nodes, reused by PHASE C. - * \return One vector of line indices per bundle. + * \param[in] config - Definition of the particular problem. + * \param[in] mixedBC - Nodes that must stay on their own, from FindMixedBoundaryNodes. + * \return One vector of indices into seeds.node per patch. */ - vector> BundleImplicitLines(const vector>& lines, - const CGeometry* fine_grid, const CConfig* config, - vector>& adj) const; + vector> BuildFrontPatches(const CFrontSeeds& seeds, const CGeometry* fine_grid, + const CConfig* config, const vector& mixedBC) const; public: /*--- This is to suppress Woverloaded-virtual, omitting it has no negative impact. ---*/ diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 22834686447..cc57de8eab0 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1125,20 +1125,21 @@ struct CMGOptions { bool MG_Smooth_Output{false}; /*!< \brief Output compact per-cycle smoothing summary. */ su2double MG_Smooth_StagnationTol{0.0}; /*!< \brief Stagnation early exit: stop if current_rms >= prev_rms * tol. 0 = disabled. */ bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ - unsigned long MG_Implicit_Lines_MaxLength{20}; /*!< \brief Maximum nodes on a wall-normal implicit line (including wall seed). */ - bool MG_Implicit_Lines_Isotropic{false}; /*!< \brief Use isotropic (vs anisotropic) agglomeration along implicit lines. */ + unsigned long MG_Implicit_Lines_MaxLength{0}; /*!< \brief Safety cap on paving stack depth in layers, 0 for none. */ unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse CV tangential to the wall. 0 = dimension-appropriate default (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ - su2double MG_Implicit_Lines_Min_AR{2.0}; /*!< \brief Smallest local cell aspect ratio for which a node still counts as - part of a stretched layer. Ends a line where the mesh stops being - stretched along it, and decides which boundaries carry a layer normal - to them. See CMultiGridGeometry::AgglomerateImplicitLines. */ - su2double MG_Implicit_Lines_Iso_AR{0.0}; /*!< \brief Aspect ratio below which a stack switches from semi-coarsening - (one fine layer per coarse CV, preserving the line) to full coarsening - (two fine layers per coarse CV). 0 disables the switch, leaving - MG_IMPLICIT_LINES_ISOTROPIC in charge for the whole stack. - See CMultiGridGeometry::AgglomerateImplicitLines. */ + su2double MG_Implicit_Lines_Min_AR{2.0}; /*!< \brief Smallest local cell aspect ratio for which a node still counts + as part of a stretched layer. Decides which non-wall boundaries carry + a layer normal to them and may therefore seed paving fronts. It is a + SEEDING gate only and never stops a front that has started. + See CMultiGridGeometry::SeedFrontNodes. */ + su2double MG_Boundary_Thicken_AR{0.0}; /*!< \brief Local aspect ratio, measured along the boundary normal, at or above + which a boundary coarse CV is left as a flat surface patch instead of + being thickened into the interior. Below it the boundary sits in mesh + that is not stretched normal to itself and the CV is grown inwards to + the full agglomeration size. 0 disables thickening altogether, which is + the historical behaviour. See CMultiGridGeometry::CMultiGridGeometry. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 6b7aca7fbc6..6a802905042 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2082,25 +2082,30 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); - /*!\brief MG_IMPLICIT_LINES_MAX_LENGTH\n DESCRIPTION: Maximum number of nodes on a wall-normal implicit agglomeration line (including the wall seed node). DEFAULT: 20 \ingroup Config*/ - addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 20); - /*!\brief MG_IMPLICIT_LINES_ISOTROPIC\n DESCRIPTION: Use isotropic agglomeration along implicit lines (4 cells per coarse CV) instead of anisotropic (2 cells per coarse CV). DEFAULT: NO \ingroup Config*/ - addBoolOption("MG_IMPLICIT_LINES_ISOTROPIC", MGOptions.MG_Implicit_Lines_Isotropic, false); + /*!\brief MG_IMPLICIT_LINES_MAX_LENGTH\n DESCRIPTION: Safety cap on how many layers deep a paving front + * may go. A front is meant to run until it reaches a boundary or the mesh stops offering a layer + * topologically identical to the one below it, so this is off by default. DEFAULT: 0 (no cap) \ingroup Config*/ + addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 0); /*!\brief MG_IMPLICIT_LINES_MAX_GROUP\n DESCRIPTION: Maximum number of parallel implicit lines merged tangential to * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_GROUP", MGOptions.MG_Implicit_Lines_Max_Group, 0); - /*!\brief MG_IMPLICIT_LINES_MIN_AR\n DESCRIPTION: Smallest local cell aspect ratio for which a node still counts as part - * of a stretched layer, measured from the ratio of dual-grid edge weights. Ends an implicit line where the mesh stops - * being stretched along it, instead of letting it run to the far field, and decides which boundaries carry a layer - * normal to them and may therefore seed lines. 1.0 disables both tests. DEFAULT: 2.0 \ingroup Config*/ + /*!\brief MG_IMPLICIT_LINES_MIN_AR\n DESCRIPTION: Smallest local cell aspect ratio for which a node still counts as + * part of a stretched layer, measured from the ratio of dual-grid edge weights. Decides which non-wall boundaries + * carry a layer normal to them and may therefore seed paving fronts; viscous walls always seed. This is a seeding + * gate only and never stops a front that has started. 1.0 lets every boundary seed. DEFAULT: 2.0 \ingroup Config*/ addDoubleOption("MG_IMPLICIT_LINES_MIN_AR", MGOptions.MG_Implicit_Lines_Min_AR, 2.0); - /*!\brief MG_IMPLICIT_LINES_ISO_AR\n DESCRIPTION: Local cell aspect ratio at which a stack of coarse CVs switches from - * semi-coarsening to full coarsening. While the mesh at the current height is stretched by more than this, one fine - * layer goes into each coarse CV, so the coarse grid keeps the wall-normal line structure a line-implicit smoother - * relies on. Once the ratio drops below it the layers are taken two at a time and the coarsening becomes isotropic. - * 0.0 disables the switch and MG_IMPLICIT_LINES_ISOTROPIC decides for the whole stack. DEFAULT: 0.0 \ingroup Config*/ - addDoubleOption("MG_IMPLICIT_LINES_ISO_AR", MGOptions.MG_Implicit_Lines_Iso_AR, 0.0); + /*!\brief MG_BOUNDARY_THICKEN_AR\n DESCRIPTION: Grow boundary coarse CVs into the interior instead of leaving them as + * surface patches one fine cell thick. The boundary agglomeration can only ever merge points that lie on the boundary + * themselves, so a boundary coarse CV comes out flat: 2x2 nodes on a surface, 2 on a ridge, never the 2x2x2 block the + * interior pass builds. Those CVs are a large share of the coarse grid by count while holding very few nodes each. + * Thickening keeps the surface footprint the boundary agglomeration chose and only adds the layer underneath it, so + * the CV still never straddles two boundary conditions. It is skipped where the mesh carries a stretched layer normal + * to the boundary, i.e. where the local aspect ratio measured along the boundary normal reaches this value, since + * there the flat CV is deliberate semi-coarsening that preserves the wall-normal resolution of a boundary layer. + * Raising it thickens more boundaries, lowering it fewer. 0.0 disables thickening entirely. DEFAULT: 0.0 + * \ingroup Config*/ + addDoubleOption("MG_BOUNDARY_THICKEN_AR", MGOptions.MG_Boundary_Thicken_AR, 0.0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); @@ -3091,6 +3096,13 @@ void CConfig::SetConfig_Options() { * weights come out uniform, and the partitioning is the same as with no weights at all. 0 disables the weights. * DEFAULT: 0 */ addDoubleOption("PARMETIS_ANISO_WEIGHT", ParMETIS_anisoWgt, 0.0); + /*!\brief PARMETIS_COLUMN_PARTITION\n DESCRIPTION: Contract each wall-normal column of stretched cells into a single + * graph vertex before partitioning, and give every node of a column the colour of its column. A partition boundary + * can then never cross a column, which is what the implicit-line agglomeration and line-implicit smoothing need, and + * the graph ParMETIS actually cuts is the wall surface. Columns are found as connected components of the edges that + * are short at both of their ends, so an isotropic mesh contracts to itself and partitions exactly as before. + * DEFAULT: NO \ingroup Config*/ + addBoolOption("PARMETIS_COLUMN_PARTITION", ParMETIS_columnPart, false); /*--- options that are used in the Hybrid RANS/LES Simulations ---*/ /*!\par CONFIG_CATEGORY:Hybrid_RANSLES Options\ingroup Config*/ diff --git a/Common/src/geometry/CGeometry.cpp b/Common/src/geometry/CGeometry.cpp index 981ac966c66..dab1417895e 100644 --- a/Common/src/geometry/CGeometry.cpp +++ b/Common/src/geometry/CGeometry.cpp @@ -4340,7 +4340,8 @@ void CGeometry::ColorMGLevels(unsigned short nMGLevels, const CGeometry* const* const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) const { auto& li = lineletInfo; - if (!li.linelets.empty() || nPoint == 0) return li; + if (li.built || nPoint == 0) return li; + li.built = true; li.lineletIdx.resize(nPoint, CLineletInfo::NO_LINELET); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 98172777442..a9c29f23961 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -88,15 +88,74 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- STEP 0: agglomerate the stretched layer above viscous walls along implicit lines, wall CV - * included. This runs before the general boundary agglomeration below so that the wall control - * volume and the layers stacked on top of it share one footprint; letting the general scheme - * claim the wall first would fix a footprint chosen without any knowledge of the lines, and the - * stack above it could then only be misaligned with its own base. Everything it claims is + /*--- STEP 0: pave the domain with advancing fronts rising from the boundaries, wall CV included. + * This runs before the general boundary agglomeration below so that the wall control volume and + * the layers stacked on top of it share one footprint; letting the general scheme claim the wall + * first would fix a footprint chosen without any knowledge of the fronts, and the stack above it + * could then only be misaligned with its own base. Everything it claims is * already marked agglomerated, so the boundary and interior passes below simply skip it. ---*/ + const auto starting_idx_lines_DBG = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config); } + const auto idx_after_lines_DBG = Index_CoarseCV; + + /*--- Points carrying a physical boundary condition. SEND_RECEIVE is not one: it only records that + * the point is mirrored on another rank. Used below to tell a genuine interior point, which a + * boundary CV may absorb, from a point on another boundary, which it may not. ---*/ + vector onPhysBoundary(fine_grid->GetnPoint(), 0); + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) + onPhysBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; + } + /*--- Nodes where two different boundary conditions meet. Nishikawa's rules never agglomerate + * these, and the rule has to hold for every phase or the same node is treated one way by the + * paving and another here. In 2D the corner test below already refused them; in 3D a ridge of + * such nodes carries one identical marker PAIR all along it and would otherwise pair up with + * itself quite happily. ---*/ + const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); + + vector bmembers; + vector nThicken_DBG(fine_grid->GetnMarker(), 0), nFlat_DBG(fine_grid->GetnMarker(), 0); + + /*--- Whether a boundary coarse CV may be grown into the interior, see MG_BOUNDARY_THICKEN_AR. The + * measurement is only needed when it can be, so a run that leaves the option off pays nothing. ---*/ + const su2double thickenAR = config->GetMGOptions().MG_Boundary_Thicken_AR; + const bool THICKEN = (thickenAR > 0.0); + const CNodeStiffness boundStiff = THICKEN ? ComputeNodeStiffness(fine_grid) : CNodeStiffness(); + + /*--- True where the mesh carries a stretched layer running normal to this boundary, the situation a + * flat boundary CV exists to preserve. Both halves of the test matter, and testing the aspect + * ratio alone is what made a fixed threshold useless on a real mesh: the ratio is undirected, so + * a boundary lying in mesh that is merely graded ALONG itself - a symmetry plane with streamwise + * stretching, say - reads as strongly stretched and never gets thickened, even though nothing + * normal to it would be lost. Requiring the stiffest direction at the node to line up with the + * boundary normal separates the two: only a boundary with cells stacked against it qualifies. ---*/ + constexpr su2double THICKEN_ANGLE_DEG = 20.0; + const su2double thickenCos = cos(THICKEN_ANGLE_DEG * PI_NUMBER / 180.0); + + auto boundaryHasLayer = [&](unsigned long iPoint, unsigned short iMarker) { + const auto jStiffest = boundStiff.jStiffest[iPoint]; + if (jStiffest == std::numeric_limits::max()) return false; + if (boundStiff.AspectRatio(iPoint) < thickenAR) return false; + + const long iVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); + if (iVertex == -1) return false; + su2double normal[MAXNDIM] = {0.0}; + fine_grid->vertex[iMarker][iVertex]->GetNormal(normal); + const su2double nrm = GeometryToolbox::Norm(nDim, normal); + if (nrm <= 0.0) return false; + + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jStiffest), fine_grid->nodes->GetCoord(iPoint), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) return false; + + su2double dot = 0.0; + for (unsigned short d = 0; d < nDim; ++d) dot += (vec[d] / len) * (normal[d] / nrm); + return fabs(dot) >= thickenCos; + }; /*--- STEP 1: The first step is the boundary agglomeration. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { @@ -130,6 +189,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- We add the seed point (child) to the parent control volume ---*/ nodes->SetChildren_CV(Index_CoarseCV, 0, iPoint); + bmembers.clear(); + bmembers.push_back(iPoint); bool agglomerate_seed = false; auto counter = 0; unsigned short copy_marker[3] = {}; @@ -228,6 +289,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (counter > 2) agglomerate_seed = false; + /*--- ...and so is a node where two markers of DIFFERENT type meet, whatever the count and + * whatever the dimension. A coarse CV holding such a node would average two conditions the + * fine grid applies separately. ---*/ + if (mixedBC[iPoint]) agglomerate_seed = false; + /*--- If the seed (parent) can be agglomerated, we try to agglomerate connected childs to the parent ---*/ /*--- Note that in 2D we allow a maximum of 4 nodes to be agglomerated ---*/ @@ -237,7 +303,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : fine_grid->nodes->GetPoints(iPoint)) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -246,17 +312,22 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); nChildren++; + bmembers.push_back(CVPoint); /*--- In 2D, we agglomerate exactly 2 nodes if the nodes are on the line edge. ---*/ if ((nDim == 2) && (counter == 1)) break; /*--- In 3D, we agglomerate exactly 2 nodes if the nodes are on the surface edge. ---*/ if ((nDim == 3) && (counter == 2)) break; /*--- Apply maxAgglomSize limit for 3D internal boundary face nodes (counter==1 in 3D). ---*/ - if (nChildren == maxAgglomSize) break; + if (nChildren >= maxAgglomSize) break; } } - /*--- Only take into account indirect neighbors for 3D faces, not 2D. ---*/ - if (nDim == 3) { + /*--- Only take into account indirect neighbors for 3D faces, not 2D. The size test has to be + * made on entry as well: the sweep above leaves with the CV exactly full whenever it hit + * the limit, and without a guard here the first indirect candidate pushed it to nine + * children, after which the equality test below could never match again and the CV grew + * without a bound at all. ---*/ + if ((nDim == 3) && (nChildren < maxAgglomSize)) { Suitable_Indirect_Neighbors.clear(); if (fine_grid->nodes->GetAgglomerate_Indirect(iPoint)) @@ -267,7 +338,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : Suitable_Indirect_Neighbors) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -282,10 +353,59 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); nChildren++; + bmembers.push_back(CVPoint); /*--- Apply maxAgglomSize limit for 3D internal boundary face nodes. ---*/ - if (nChildren == maxAgglomSize) break; + if (nChildren >= maxAgglomSize) break; + } + } + } + + /*--- Thicken the surface patch into the interior. Everything above only ever considers + * candidates that lie on the boundary themselves, because SetBoundAgglomeration refuses + * an interior point outright, so a boundary coarse CV comes out as a film one fine cell + * thick: 2x2 nodes at best on a surface, 2 on a ridge, never the 2x2x2 block the domain + * pass builds everywhere else. That is a coarse CV of 4 nodes where 8 were available, + * and since every boundary of the mesh is covered in them they make up a large share of + * the coarse grid by count while holding very few nodes each. + * + * Growing into the interior fixes that without touching which surface nodes belong + * together: the footprint on the boundary is already decided above, this only adds the + * layer underneath it. Candidates are restricted to genuinely interior points, so the CV + * still cannot straddle two boundary conditions, and the node that joins is the one + * sharing the most faces with what the CV already holds - the same rule STEP 2 uses, + * which is what makes it close into blocks rather than grow into stars. ---*/ + const bool thickenThis = THICKEN && !boundaryHasLayer(iPoint, iMarker); + (thickenThis ? nThicken_DBG : nFlat_DBG)[iMarker]++; + + while (thickenThis && (nChildren < maxAgglomSize)) { + auto best = std::numeric_limits::max(); + unsigned short best_shared = 0; + + for (auto mPoint : bmembers) { + for (auto jPoint : fine_grid->nodes->GetPoints(mPoint)) { + if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; + if (!fine_grid->nodes->GetDomain(jPoint)) continue; + if (onPhysBoundary[jPoint]) continue; + if (!GeometricalCheck(jPoint, fine_grid, config)) continue; + + unsigned short shared = 0; + for (auto kPoint : fine_grid->nodes->GetPoints(jPoint)) + shared += (find(bmembers.begin(), bmembers.end(), kPoint) != bmembers.end()); + + if (shared > best_shared) { + best_shared = shared; + best = jPoint; + } } } + + if (best == std::numeric_limits::max()) break; + + fine_grid->nodes->SetParent_CV(best, Index_CoarseCV); + if (fine_grid->nodes->GetAgglomerate_Indirect(best)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); + nodes->SetChildren_CV(Index_CoarseCV, nChildren, best); + nChildren++; + bmembers.push_back(best); } } @@ -352,8 +472,24 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * other option still touches one, so it wins and completes the square; the same argument then * repeats one axis up and completes the cube. On a structured hex mesh the result is an exact * 2x2x2 block, which is what the implicit-line stacks already produce and what the isotropic - * region should match. Ties are settled by distance to the centroid of the current members, which - * keeps the growth compact on meshes with no preferred axis to reason about. + * region should match. + * + * Ties are settled by distance to the centroid, but that distance has to be measured in cells and + * not in metres. Until the CV holds an L there is nothing for the shared count to prefer - on a hex + * graph the node diagonal to two members is not adjacent to the seed, so every candidate shares + * exactly one face and the distance decides alone. In a stretched cell the neighbour across the + * thin direction is nearer than any neighbour along the layer by whatever the aspect ratio happens + * to be, so the CV steps that way, and then finds the next step in the SAME direction nearer still. + * It walks the boundary layer end to end: measured on a real mesh, 63% of the CVs built here came + * out as eight nodes in a straight line. That is the wrong shape, and worse, the wrong direction to + * coarsen in, since it merges exactly the wall-normal cells the implicit lines exist to keep apart. + * + * The yardstick is the seed's own incident edges: a candidate's offset is divided by the length of + * the edge pointing most nearly the same way. A step across the layer and a step along it then both + * come to about one, whatever the stretching, and the shared count takes over from there. Choosing + * the edge by direction rather than by which Cartesian axis it is closest to is what keeps this + * usable on a curved boundary, where the wall-normal direction is not any one axis and a per-axis + * spacing would be measuring the wrong thing over most of the surface. * * The candidate set grows with the CV rather than being fixed to the seed's own neighbours: the * far corner of a cube is not adjacent to the seed, it only becomes reachable once the nodes @@ -366,6 +502,19 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un vector members, candidates; members.reserve(maxAgglomSize); + /*--- A local frame at the current seed, up to nDim of its incident edges chosen to be as mutually + * orthogonal as possible, each with its own length. It is the yardstick described above: an + * offset is resolved onto these directions and each component divided by that direction's own + * spacing. Being built from the edges themselves it turns with the mesh, so it still measures + * the wall-normal direction correctly where a boundary curves away from any Cartesian axis, and + * on an axis-aligned mesh it reduces to dividing x, y and z by their own spacings. ---*/ + vector> frameDir, edgeDir; + vector frameLen, edgeLen; + vector edgeUsed; + + const auto idx_after_bound_DBG = Index_CoarseCV; + unsigned long nRejectedSeed_DBG = 0; + auto iteration = 0ul; while (!MGQueue_InnerCV.EmptyQueue() && (iteration < fine_grid->GetnPoint())) { const auto iPoint = MGQueue_InnerCV.NextCV(); @@ -405,6 +554,58 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un addMember(iPoint); + edgeDir.clear(); + edgeLen.clear(); + su2double seedShortest = std::numeric_limits::max(); + for (auto jPoint : fine_grid->nodes->GetPoints(iPoint)) { + su2double e[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(iPoint), e); + const su2double len = GeometryToolbox::Norm(nDim, e); + if (len <= 0.0) continue; + std::array u{}; + for (unsigned short d = 0; d < nDim; ++d) u[d] = e[d] / len; + edgeDir.push_back(u); + edgeLen.push_back(len); + seedShortest = std::min(seedShortest, len); + } + /*--- A seed with no usable edge cannot be measured against anything, leave the scale at one. ---*/ + if (seedShortest == std::numeric_limits::max()) seedShortest = 1.0; + + /*--- The shortest edge goes in first, so that the direction the mesh is stretched along is the + * one measured against its own small spacing; the rest are taken in order of how orthogonal + * they are to what is already in the frame. ---*/ + frameDir.clear(); + frameLen.clear(); + edgeUsed.assign(edgeDir.size(), 0); + while (frameDir.size() < nDim) { + long pick = -1; + su2double bestScore = -1.0; + for (size_t i = 0; i < edgeDir.size(); ++i) { + if (edgeUsed[i]) continue; + su2double score; + if (frameDir.empty()) { + score = 1.0 / edgeLen[i]; + } else { + score = 2.0; + for (const auto& f : frameDir) { + su2double a = 0.0; + for (unsigned short d = 0; d < nDim; ++d) a += f[d] * edgeDir[i][d]; + score = std::min(score, 1.0 - fabs(a)); + } + } + if (score > bestScore) { + bestScore = score; + pick = static_cast(i); + } + } + if (pick < 0) break; + edgeUsed[pick] = 1; + frameDir.push_back(edgeDir[pick]); + frameLen.push_back(edgeLen[pick]); + } + /*--- Without a full frame the offset cannot be resolved, fall back to one isotropic scale. ---*/ + const bool haveFrame = (frameDir.size() == nDim); + while (nChildren < maxAgglomSize) { /*--- Centroid of what the CV holds so far, used only to break ties. ---*/ su2double centroid[MAXNDIM] = {0.0}; @@ -423,8 +624,24 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned short shared = 0; for (auto jPoint : fine_grid->nodes->GetPoints(CVPoint)) shared += inCV[jPoint]; - const su2double dist = - GeometryToolbox::SquaredDistance(nDim, fine_grid->nodes->GetCoord(CVPoint), centroid); + /*--- Distance to the centroid in cells rather than in metres: the offset is scaled by the + * seed edge pointing most nearly along it. ---*/ + su2double off[MAXNDIM] = {0.0}; + for (unsigned short d = 0; d < nDim; ++d) off[d] = fine_grid->nodes->GetCoord(CVPoint)[d] - centroid[d]; + const su2double offLen = GeometryToolbox::Norm(nDim, off); + + su2double dist = 0.0; + if (haveFrame) { + for (size_t k = 0; k < frameDir.size(); ++k) { + su2double p = 0.0; + for (unsigned short d = 0; d < nDim; ++d) p += off[d] * frameDir[k][d]; + const su2double q = p / frameLen[k]; + dist += q * q; + } + } else { + const su2double r = offLen / seedShortest; + dist = r * r; + } if ((shared > best_shared) || ((shared == best_shared) && (dist < best_dist))) { best = CVPoint; @@ -448,10 +665,13 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- The seed point can not be agglomerated because of size, domain, streching, etc. move the point to the lowest priority ---*/ + nRejectedSeed_DBG++; MGQueue_InnerCV.MoveCV(iPoint, -1); } } + const auto idx_after_domain_DBG = Index_CoarseCV; + /*--- Convert any point that was not agglomerated into a coarse point. ---*/ for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { @@ -464,6 +684,81 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- TEMPORARY DIAGNOSTIC: size distribution of the coarse CVs, by the phase that created them. + * + * The child count on its own does not say what a CV looks like: eight children can be the 2x2x2 + * block that is wanted, or a 4x2x1 slab, or a 1x8 strip, and in a planar slice through the mesh + * those are indistinguishable from a CV that really is small - a cube shows only its four nodes + * that lie in the slice, a slab edge-on shows two. Counting the fine edges whose two ends fall in + * the same CV separates them without reference to any coordinate direction: a 2x2x2 block has 12 + * internal edges, a flat 2x2 has 4, a 1x4 strip has 3, a pair has 1. So "8 children, 12 edges" is + * a cube and "8 children, 10 edges" is not. ---*/ + vector intEdges_DBG(Index_CoarseCV, 0); + for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + const auto pi = fine_grid->nodes->GetParent_CV(iPoint); + if (pi >= Index_CoarseCV) continue; + for (auto jPoint : fine_grid->nodes->GetPoints(iPoint)) { + if (jPoint <= iPoint) continue; + if (!fine_grid->nodes->GetDomain(jPoint)) continue; + if (fine_grid->nodes->GetParent_CV(jPoint) == pi) intEdges_DBG[pi]++; + } + } + + { + auto histOf = [&](unsigned long lo, unsigned long hi, const char* name) { + unsigned long h[10] = {0}, tot = 0, nod = 0; + unsigned long cube8 = 0, noncube8 = 0, square4 = 0, strip4 = 0; + unsigned long e8[14] = {0}; + for (auto c = lo; c < hi; ++c) { + const auto n = nodes->GetnChildren_CV(c); + const auto e = intEdges_DBG[c]; + if (n == 8) { + ((e >= 12) ? cube8 : noncube8)++; + e8[std::min(e, 13)]++; + } + if (n == 4) ((e >= 4) ? square4 : strip4)++; + h[std::min(n, 9)]++; + tot++; + nod += n; + } + if (tot == 0) return; + cout << " " << name << ": " << tot << " CVs, " << nod << " nodes, avg " << (su2double(nod) / su2double(tot)) + << " sizes"; + for (unsigned s = 1; s <= 9; ++s) + if (h[s] > 0) cout << " " << s << ":" << h[s]; + if (h[8] > 0) { + cout << " [of the 8s: " << cube8 << " are 2x2x2 cubes, " << noncube8 << " are slabs/strips; internal edges"; + for (unsigned e = 7; e <= 13; ++e) + if (e8[e] > 0) cout << " " << (e == 13 ? ">=13" : to_string(e)) << ":" << e8[e]; + cout << "]"; + } + if (h[4] > 0) cout << " [of the 4s: " << square4 << " square, " << strip4 << " strip]"; + cout << endl; + }; + unsigned long degSum = 0, degN = 0; + for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + degSum += fine_grid->nodes->GetnPoint(iPoint); + degN++; + } + cout << " CV size distribution by phase (maxAgglomSize=" << maxAgglomSize + << ", mean fine-graph degree=" << (su2double(degSum) / su2double(max(degN, 1ul))) + << ", a 2x2x2 block has 12 internal edges):" << endl; + histOf(0, starting_idx_lines_DBG, "pre-lines "); + histOf(starting_idx_lines_DBG, idx_after_lines_DBG, "implicit lines "); + histOf(idx_after_lines_DBG, idx_after_bound_DBG, "boundary STEP1 "); + histOf(idx_after_bound_DBG, idx_after_domain_DBG, "domain STEP2 "); + histOf(idx_after_domain_DBG, Index_CoarseCV, "leftover single"); + cout << " STEP2 rejected seeds: " << nRejectedSeed_DBG << ", iterations used: " << iteration << "/" + << fine_grid->GetnPoint() << endl; + for (auto m = 0u; m < fine_grid->GetnMarker(); ++m) { + if (nThicken_DBG[m] + nFlat_DBG[m] == 0) continue; + cout << " marker " << config->GetMarker_All_TagBound(m) << ": thickened " << nThicken_DBG[m] << ", kept flat " + << nFlat_DBG[m] << endl; + } + } + nPointDomain = Index_CoarseCV; nPoint = nPointDomain; @@ -480,6 +775,21 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, but only for the CVs that really do border another rank rather than for every CV near one. ---*/ + /*--- Coarse CVs that must be left exactly as the agglomeration made them: those holding a node + * where two different boundary conditions meet. Both repair passes below exist to get rid of + * one-child control volumes, and a deliberately isolated junction IS a one-child control volume, + * so without this they undo the isolation - the wall/symmetry node of a flat plate came out + * correctly alone and was then merged straight back into the CV above it. They have to be + * protected as a TARGET as well as a source: pass two merges a singleton into its smallest + * neighbour, and a one-child junction CV is by construction the smallest neighbour there is. ---*/ + vector mustStayAlone(nPointDomain, false); + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) + for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) + if (mixedBC[nodes->GetChildren_CV(iCoarsePoint, iChildren)]) { + mustStayAlone[iCoarsePoint] = true; + break; + } + vector touchesPartition(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -495,10 +805,12 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { + if (mustStayAlone[iCoarsePoint]) continue; if ((nodes->GetnPoint(iCoarsePoint) == 1) && !touchesPartition[iCoarsePoint]) { /*--- Find the neighbor of the isolated point. This neighbor is the right control volume ---*/ const auto iCoarsePoint_Complete = nodes->GetPoint(iCoarsePoint, 0); + if (mustStayAlone[iCoarsePoint_Complete]) continue; /*--- Check if merging would exceed the maximum agglomeration size ---*/ auto nChildren_Target = nodes->GetnChildren_CV(iCoarsePoint_Complete); @@ -512,6 +824,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint_Complete)) { if (nChildrenToRedistribute == 0) break; + if (mustStayAlone[jCoarsePoint]) continue; auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); if (nChildren_Neighbor < maxAgglomSize) { @@ -590,6 +903,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; + if (mustStayAlone[iCoarsePoint]) continue; if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the @@ -601,6 +915,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long best_neighbor = std::numeric_limits::max(); unsigned short best_nChildren = 0; for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { + if (mustStayAlone[jCoarsePoint]) continue; const auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); /*--- Skip neighbors already emptied by an earlier merge in this same pass. ---*/ if (nChildren_Neighbor == 0) continue; @@ -898,6 +1213,20 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- Optional dump of the finished agglomeration: one line per owned fine point, "x y z parentCV". + * It has to come from the END of the constructor, after both repair passes and the renumbering, + * because those still move fine points between coarse CVs - a dump taken before them shows what + * the agglomeration intended rather than what the solver will actually use. ---*/ + if (getenv("DUMPAGGLOM") != nullptr) { + ofstream fdump(string("agglom_level") + to_string(iMesh) + ".dat"); + for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + const auto* c = fine_grid->nodes->GetCoord(iPoint); + fdump << c[0] << " " << c[1] << " " << (nDim == 3 ? c[2] : 0.0) << " " << fine_grid->nodes->GetParent_CV(iPoint) + << "\n"; + } + } + edgeColorGroupSize = config->GetEdgeColoringGroupSize(); } @@ -918,8 +1247,35 @@ bool CMultiGridGeometry::GeometricalCheck(unsigned long iPoint, const CGeometry* return (Volume); } +vector CMultiGridGeometry::FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const { + vector mixed(fine_grid->GetnPoint(), 0); + + /*--- The first physical condition seen at each node, -1 until one is. A second one of a different + * KIND is what makes the node mixed. A second marker of the same kind - two wall patches meeting + * - is not, and neither is SEND_RECEIVE, which records where the partition runs and says nothing + * about the boundary condition. ---*/ + vector firstBC(fine_grid->GetnPoint(), -1); + + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + const auto bc = static_cast(config->GetMarker_All_KindBC(iMarker)); + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { + const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (firstBC[iPoint] < 0) + firstBC[iPoint] = bc; + else if (firstBC[iPoint] != bc) + mixed[iPoint] = 1; + } + } + return mixed; +} + bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, - const CGeometry* fine_grid, const CConfig* config) const { + const CGeometry* fine_grid, const CConfig* config, + const vector& mixedBC) const { + /*--- A node where two boundary conditions of different type meet is never merged with anything. ---*/ + if (mixedBC[CVPoint]) return false; + bool agglomerate_CV = false; /*--- Basic condition, the point has not been previously agglomerated, it belongs to the domain, @@ -1495,54 +1851,43 @@ CMultiGridGeometry::CNodeStiffness CMultiGridGeometry::ComputeNodeStiffness(cons return stiff; } -vector> CMultiGridGeometry::BuildImplicitLines(const CGeometry* fine_grid, const CConfig* config, - const CNodeStiffness& stiff) const { - /*--- Stop a line where its direction deviates by more than this from the previous step. ---*/ - constexpr su2double ANGLE_THRESHOLD_DEG = 20.0; +namespace { + +/*--- Unit normal of a boundary at a vertex, false if the marker does not reach iPoint. Boundary + * normals point INTO the domain, so this doubles as a starting direction to march in. ---*/ +bool VertexUnitNormal(const CGeometry* grid, unsigned short nDim, unsigned long iPoint, unsigned short iMarker, + su2double* unitNormal) { + const long iVertex = grid->nodes->GetVertex(iPoint, iMarker); + if (iVertex == -1) return false; + grid->vertex[iMarker][iVertex]->GetNormal(unitNormal); + const su2double nrm = GeometryToolbox::Norm(nDim, unitNormal); + if (nrm <= 0.0) return false; + for (unsigned short d = 0; d < nDim; ++d) unitNormal[d] /= nrm; + return true; +} + +} // namespace + +CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeometry* fine_grid, const CConfig* config, + const CNodeStiffness& stiff) const { /*--- Fraction of a marker's nodes that must sit in a layer before the whole marker may seed. ---*/ constexpr su2double QUALIFIED_FRACTION = 0.5; - + constexpr su2double ANGLE_THRESHOLD_DEG = 30.0; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); - const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; + const su2double MIN_AR = config->GetMGOptions().MG_Implicit_Lines_Min_AR; const bool USE_AR = (MIN_AR > 1.0); - const auto nPointFine = fine_grid->GetnPoint(); const auto nMarkerFine = fine_grid->GetnMarker(); constexpr auto NO_POINT = std::numeric_limits::max(); - vector> lines; - vector> dir; /*!< Current marching direction of each line. */ - vector claimed(nPointFine, 0); + CFrontSeeds seeds; + vector taken(fine_grid->GetnPoint(), 0); auto isWall = [&](unsigned short bc) { return (bc == HEAT_FLUX) || (bc == ISOTHERMAL) || (bc == CHT_WALL_INTERFACE) || (bc == SMOLUCHOWSKI_MAXWELL); }; - /*--- Nodes on a boundary that carries a boundary condition. A line must not grow into one: those - * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two - * boundaries. CPoint's Boundary flag cannot answer this, as it is also set by SEND_RECEIVE, so on - * a partitioned mesh it is true for ordinary interior nodes of the send fringe and every line - * would stop one layer short of the partition. Walking the markers' own vertex lists is both - * exact and cheaper than testing every point against every marker. ---*/ - vector onPhysicalBoundary(nPointFine, 0); - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; - for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) - onPhysicalBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; - } - - /*--- Unit normal of a boundary at a vertex, false if the marker does not reach iPoint. ---*/ - auto vertexNormal = [&](unsigned long iPoint, unsigned short iMarker, su2double* unitNormal) { - const long ChildVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); - if (ChildVertex == -1) return false; - fine_grid->vertex[iMarker][ChildVertex]->GetNormal(unitNormal); - const su2double nrm = GeometryToolbox::Norm(nDim, unitNormal); - if (nrm <= 0.0) return false; - for (unsigned short d = 0; d < nDim; ++d) unitNormal[d] /= nrm; - return true; - }; - /*--- True if the mesh at iPoint is stretched along the boundary normal, i.e. this boundary has a * layer growing off it the way a viscous wall does. On the side planes of a bump the mesh is just * as stretched, but in a direction running ALONG the plane, and those nodes belong to the wall. ---*/ @@ -1559,23 +1904,22 @@ vector> CMultiGridGeometry::BuildImplicitLines(const CGeom return fabs(GeometryToolbox::DotProduct(nDim, vec, unitNormal)) >= cos_threshold; }; - /*--- Seed a line at every eligible node of a marker. ---*/ auto seedMarker = [&](unsigned short iMarker, bool requireLayer) { for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); if (!fine_grid->nodes->GetDomain(iPoint)) continue; if (fine_grid->nodes->GetAgglomerate(iPoint)) continue; - if (claimed[iPoint]) continue; /*--- A node on two markers must seed only one line. ---*/ + if (taken[iPoint]) continue; /*--- A node on two markers must seed only one front. ---*/ su2double Normal[MAXNDIM] = {0.0}; - if (!vertexNormal(iPoint, iMarker, Normal)) continue; + if (!VertexUnitNormal(fine_grid, nDim, iPoint, iMarker, Normal)) continue; if (requireLayer && !hasLayerNormalTo(iPoint, Normal)) continue; - lines.push_back({iPoint}); - claimed[iPoint] = 1; - std::array d0{}; - for (unsigned short d = 0; d < nDim; ++d) d0[d] = Normal[d]; - dir.push_back(d0); + std::array n0{}; + for (unsigned short d = 0; d < nDim; ++d) n0[d] = Normal[d]; + seeds.node.push_back(iPoint); + seeds.normal.push_back(n0); + taken[iPoint] = 1; } }; @@ -1591,7 +1935,7 @@ vector> CMultiGridGeometry::BuildImplicitLines(const CGeom * fine grid does not single out. * * The verdict is taken per marker, not per node: seeding isolated qualifying nodes on a marker - * that mostly does not qualify scatters one-line bundles, i.e. coarse CVs that do not coarsen + * that mostly does not qualify scatters one-node patches, i.e. coarse CVs that do not coarsen * tangentially at all. The two populations are far apart in practice - boundaries with a layer * normal to them qualify at 100%, side planes, inlets and far fields at 13% and below - so any * threshold near a half separates them. ---*/ @@ -1605,7 +1949,7 @@ vector> CMultiGridGeometry::BuildImplicitLines(const CGeom auto canSeed = [&](unsigned short iMarker) { const auto bc = config->GetMarker_All_KindBC(iMarker); /*--- Periodic boundaries are left out: the two halves are the same physical location under a - * transform and have their own matching, which a line running into one would disturb. ---*/ + * transform and have their own matching, which a front running into one would disturb. ---*/ return (bc != SEND_RECEIVE) && (bc != PERIODIC_BOUNDARY) && !isWall(bc); }; @@ -1620,13 +1964,14 @@ vector> CMultiGridGeometry::BuildImplicitLines(const CGeom const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); if (!fine_grid->nodes->GetDomain(iPoint)) continue; su2double Normal[MAXNDIM] = {0.0}; - if (!vertexNormal(iPoint, iMarker, Normal)) continue; + if (!VertexUnitNormal(fine_grid, nDim, iPoint, iMarker, Normal)) continue; nValid[cfgOfMarker[iMarker]]++; if (hasLayerNormalTo(iPoint, Normal)) nQualified[cfgOfMarker[iMarker]]++; } } - /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. ---*/ + /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. + * Every rank reaches these collectives, including one that owns no boundary at all. ---*/ if (nMarkerCfg > 0) { vector tmp(nMarkerCfg); SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -1644,116 +1989,36 @@ vector> CMultiGridGeometry::BuildImplicitLines(const CGeom } } - /*--- Grow every line one step per sweep rather than one line to completion at a time, claiming a - * node globally the moment any line takes it. Growing them one at a time lets an early line run - * the full depth of the layer and consume nodes a neighbouring line needed, so that line stops - * after a step or two and the lengths become too uneven to bundle into columns of uniform depth. - * In lockstep all lines compete for each layer on equal terms, which on an extruded prismatic - * layer reproduces the mesh's own structure. ---*/ - vector growing(lines.size(), 1); - - for (bool any_grew = true; any_grew;) { - any_grew = false; - - for (unsigned long li = 0; li < lines.size(); ++li) { - if (!growing[li]) continue; - if (lines[li].size() >= MAX_LINE_LENGTH) { - growing[li] = 0; - continue; - } - - const auto current = lines[li].back(); - su2double best_dot = -2.0, best_dir[MAXNDIM] = {0.0}; - auto best = NO_POINT; - unsigned short best_neigh = 0; - - for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(current); ++iNeigh) { - const auto jPoint = fine_grid->nodes->GetPoint(current, iNeigh); - /*--- Halo nodes stay out: their parent is dictated by the rank that owns them and arrives - * through the MPI relay, so a line claiming one would fight that assignment. ---*/ - if (!fine_grid->nodes->GetDomain(jPoint)) continue; - if (onPhysicalBoundary[jPoint]) continue; - if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; - if (claimed[jPoint]) continue; - - su2double vec[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(current), vec); - const su2double len = GeometryToolbox::Norm(nDim, vec); - if (len <= 0.0) continue; - for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - - const su2double dot = GeometryToolbox::DotProduct(nDim, vec, dir[li].data()); - if (dot > best_dot) { - best_dot = dot; - best = jPoint; - best_neigh = iNeigh; - for (unsigned short d = 0; d < nDim; ++d) best_dir[d] = vec[d]; - } - } - - if ((best == NO_POINT) || (best_dot < cos_threshold)) { - growing[li] = 0; - continue; - } - - /*--- End the line where the mesh stops being stretched along it. Without this the only limits - * are the direction cone and MAX_LINE_LENGTH, so a line leaves the boundary layer and keeps - * stacking coarse CVs along a direction the fine grid does not single out. Taking the weight - * of this one edge over the weakest edge at the node keeps the measure directional: a mesh - * graded in the streamwise direction reads as stretched to an undirected min/max test even - * far from any wall. ---*/ - if (USE_AR && (stiff.wMin[current] > 0.0)) { - const auto jPoint = fine_grid->nodes->GetPoint(current, best_neigh); - const auto iEdge = fine_grid->nodes->GetEdge(current, best_neigh); - const su2double area = GeometryToolbox::Norm(nDim, fine_grid->edges->GetNormal(iEdge)); - const su2double w = - 0.5 * area * (1.0 / fine_grid->nodes->GetVolume(current) + 1.0 / fine_grid->nodes->GetVolume(jPoint)); - if (w / stiff.wMin[current] < MIN_AR) { - growing[li] = 0; - continue; - } - } - - for (unsigned short d = 0; d < nDim; ++d) dir[li][d] = best_dir[d]; - lines[li].push_back(best); - claimed[best] = 1; - any_grew = true; - } - } - - /*--- A line needs at least one interior node to contribute anything. ---*/ - vector> kept; - kept.reserve(lines.size()); - for (auto& L : lines) - if (L.size() >= 2) kept.push_back(std::move(L)); - - return kept; + return seeds; } -vector> CMultiGridGeometry::BundleImplicitLines(const vector>& lines, - const CGeometry* fine_grid, - const CConfig* config, - vector>& adj) const { - /*--- Every line must end up in exactly one bundle, and a bundle must be a compact patch on the wall: - * in 3D the four lines rising from the corners of one wall quadrilateral, in 2D the two lines from - * the ends of one wall edge. Choosing, for each line independently, a set of neighbours to merge - * with does not do this - the relation is not symmetric, so line 1 claiming {2,3} does not stop - * line 2 claiming {1,4}, and the bundles overlap and fight over nodes. +vector> CMultiGridGeometry::BuildFrontPatches(const CFrontSeeds& seeds, + const CGeometry* fine_grid, const CConfig* config, + const vector& mixedBC) const { + /*--- Every seed must end up in exactly one patch, and a patch must be compact: in 3D the four nodes + * of one boundary quadrilateral, in 2D the two ends of one boundary edge. Choosing, for each seed + * independently, a set of neighbours to merge with does not do this - the relation is not + * symmetric, so seed 1 claiming {2,3} does not stop seed 2 claiming {1,4}, and the patches + * overlap and fight over nodes. * - * Repeated pairwise matching avoids that by construction. One round pairs adjacent lines into the - * wall edge, a second pairs adjacent pairs into the wall quadrilateral. Each round is a matching, - * so membership stays mutually exclusive and the result is a true partition. It needs nothing but - * point-to-point connectivity, so it works identically on every multigrid level - boundary face - * connectivity does not survive agglomeration, so a literal "same quadrilateral" test would only - * ever work for the first coarsening. Two rounds reach 4, which is the 3D group size, so the - * number of rounds follows from max_group instead of iterating to a fixed point. ---*/ - const auto nLines = lines.size(); + * Repeated pairwise matching avoids that by construction. One round pairs adjacent seeds into the + * boundary edge, a second pairs adjacent pairs into the boundary quadrilateral. Each round is a + * matching, so membership stays mutually exclusive and the result is a true partition. It needs + * nothing but point-to-point connectivity, so it works identically on every multigrid level - + * boundary face connectivity does not survive agglomeration, so a literal "same quadrilateral" + * test would only ever work for the first coarsening. Two rounds reach 4, which is the 3D patch + * size, so the number of rounds follows from max_group instead of iterating to a fixed point. + * + * This patch is the ONLY thing that decides the footprint of the stack above it. The front that + * rises from it keeps exactly these nodes' successors, layer after layer, so getting the patch + * square is what makes the coarse CVs square all the way up. ---*/ + const auto nSeeds = seeds.node.size(); unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; - /*--- Marker signature of each line's boundary node, as a bitmask over the physical markers. Lines - * may only be bundled when these match, so a bundle never straddles a change of boundary - * condition - the rule ordinary agglomeration uses for ridges and valleys. ---*/ + /*--- Marker signature of each seed, as a bitmask over the physical markers. Seeds may only be + * matched when these agree, so a patch never straddles a change of boundary condition - the rule + * ordinary agglomeration uses for ridges and valleys. ---*/ const auto nMarkerFine = fine_grid->GetnMarker(); vector physBit(nMarkerFine, -1); unsigned nPhys = 0; @@ -1761,93 +2026,165 @@ vector> CMultiGridGeometry::BundleImplicitLines(const vect if (config->GetMarker_All_KindBC(iMarker) != SEND_RECEIVE) physBit[iMarker] = static_cast(nPhys++); const unsigned nWords = std::max(1u, (nPhys + 63u) / 64u); - vector sig(nLines * nWords, 0); - for (unsigned long li = 0; li < nLines; ++li) + vector sig(nSeeds * nWords, 0); + for (unsigned long si = 0; si < nSeeds; ++si) for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { if (physBit[iMarker] < 0) continue; - if (fine_grid->nodes->GetVertex(lines[li][0], iMarker) == -1) continue; + if (fine_grid->nodes->GetVertex(seeds.node[si], iMarker) == -1) continue; const auto b = static_cast(physBit[iMarker]); - sig[li * nWords + b / 64] |= (uint64_t(1) << (b % 64)); + sig[si * nWords + b / 64] |= (uint64_t(1) << (b % 64)); } - auto sameSig = [&](unsigned long la, unsigned long lb) { + auto sameSig = [&](unsigned long sa, unsigned long sb) { for (unsigned w = 0; w < nWords; ++w) - if (sig[la * nWords + w] != sig[lb * nWords + w]) return false; + if (sig[sa * nWords + w] != sig[sb * nWords + w]) return false; return true; }; - /*--- Line adjacency, inherited from the boundary nodes' mesh connectivity. ---*/ - vector lineOfWallNode(fine_grid->GetnPoint(), -1); - for (unsigned long li = 0; li < nLines; ++li) lineOfWallNode[lines[li][0]] = static_cast(li); + /*--- Seed-to-seed adjacency, inherited from the boundary nodes' mesh connectivity. ---*/ + vector seedOfNode(fine_grid->GetnPoint(), -1); + for (unsigned long si = 0; si < nSeeds; ++si) seedOfNode[seeds.node[si]] = static_cast(si); - adj.assign(nLines, {}); - for (unsigned long li = 0; li < nLines; ++li) - for (auto jPoint : fine_grid->nodes->GetPoints(lines[li][0])) { - const auto lj = lineOfWallNode[jPoint]; - if ((lj >= 0) && (static_cast(lj) != li)) adj[li].push_back(static_cast(lj)); + vector> adj(nSeeds); + for (unsigned long si = 0; si < nSeeds; ++si) + for (auto jPoint : fine_grid->nodes->GetPoints(seeds.node[si])) { + const auto sj = seedOfNode[jPoint]; + if ((sj >= 0) && (static_cast(sj) != si)) adj[si].push_back(static_cast(sj)); } - vector> groups(nLines); - vector groupOf(nLines); - for (unsigned long li = 0; li < nLines; ++li) { - groups[li] = {li}; - groupOf[li] = li; + /*--- Global point index of each seed, used everywhere below as the deterministic sort key. Local + * indices depend on the partitioning, so ordering by them would make the coarse grid depend on + * the rank count. ---*/ + vector sgkey(nSeeds); + for (unsigned long si = 0; si < nSeeds; ++si) sgkey[si] = fine_grid->nodes->GetGlobalIndex(seeds.node[si]); + + vector> groups; + vector groupOf(nSeeds); + groups.reserve(nSeeds); + + /*--- Seeds that sit next to an isolated junction node. A run of seeds between two junctions has + * whatever length the geometry gives it, and when that length is odd one seed cannot pair. Left + * to the plain index order the odd one out always lands at the END of the run, because that is + * where the sweep runs out - and the end of a run IS a junction. The result is a one-wide stack + * rising the full height of the mesh immediately beside the junction, so the coarse grid reads + * [4][4][2][1] on one side of it and [1][4][4] on the other. On a flat plate that puts the + * defect against the leading edge, which is the last place it should be. + * + * Pairing these first pushes the odd one out into the interior of the run, where a slightly + * narrower stack costs nothing, and leaves both sides of every junction matching. ---*/ + vector nextToMixed(nSeeds, 0); + for (unsigned long si = 0; si < nSeeds; ++si) { + if (mixedBC[seeds.node[si]]) continue; + for (auto jPoint : fine_grid->nodes->GetPoints(seeds.node[si])) + if (mixedBC[jPoint]) { + nextToMixed[si] = 1; + break; + } + } + + /*--- Every seed starts as its own group; the rounds below merge them. ---*/ + for (unsigned long si = 0; si < nSeeds; ++si) { + groupOf[si] = si; + groups.push_back({si}); } const unsigned nRounds = (max_group <= 1) ? 0 : ((max_group <= 2) ? 1 : 2); + + /*--- One admissible merge of two groups, weighted by how many seed-to-seed adjacencies they share. + * A group lying ALONGSIDE this one touches it along its whole length and shares two, whereas one + * continuing in the same direction touches at an end and shares one. So weight 2 is the square + * and weight 1 is the strip, and the strip extrudes into a coarse CV elongated in one + * boundary-tangential direction. ---*/ + struct CMerge { + unsigned long g, h; /*!< \brief The two groups, g < h. */ + unsigned long weight; /*!< \brief Shared adjacencies: 2 makes a square, 1 makes a strip. */ + unsigned long keyG, keyH; /*!< \brief Their global-index keys, the deterministic tie-break. */ + }; + + vector merges; vector> shared; + vector gkey; + vector consumed; for (unsigned round = 0; round < nRounds; ++round) { - vector consumed(groups.size(), 0); - vector> merged; - merged.reserve(groups.size()); - - for (unsigned long g = 0; g < groups.size(); ++g) { - if (consumed[g]) continue; - consumed[g] = 1; - auto group = groups[g]; - - /*--- Count how many line-to-line adjacencies this group shares with each candidate. Merging the - * candidate that shares the most keeps the patch square: a pair lying alongside this one - * touches it along its whole length and shares two adjacencies, whereas a pair continuing in - * the same direction touches at one end and shares one. Taking the first candidate that fits - * instead, as this did, produces a 1x4 strip of lines about as often as the mesh offers one, - * and those extrude into coarse CVs elongated in one wall-tangential direction. ---*/ + const auto nGroups = groups.size(); + + auto groupNextToMixed = [&](unsigned long g) { + for (auto si : groups[g]) + if (nextToMixed[si]) return true; + return false; + }; + + /*--- Sort key of each group: the smallest global point index it holds. ---*/ + gkey.assign(nGroups, std::numeric_limits::max()); + for (unsigned long g = 0; g < nGroups; ++g) + for (auto si : groups[g]) gkey[g] = std::min(gkey[g], sgkey[si]); + + /*--- Every merge this round could make. Counted once per unordered pair: adjacency is symmetric, + * so the count from g's side equals the count from h's, and taking only h > g avoids both. ---*/ + merges.clear(); + for (unsigned long g = 0; g < nGroups; ++g) { + /*--- A node where two different boundary conditions meet stays a patch of its own, so the front + * rising from it is one node wide and its coarse CVs never reach across the junction. Both + * sides of the merge are tested: skipping only the mixed group would still let an ordinary + * group reach out and take it. ---*/ + if (mixedBC[seeds.node[groups[g].front()]]) continue; shared.clear(); - for (auto li : group) - for (auto lj : adj[li]) { - const auto h = groupOf[lj]; - if ((h == g) || consumed[h]) continue; - if (group.size() + groups[h].size() > max_group) continue; - if (!sameSig(groups[h].front(), group.front())) continue; + for (auto si : groups[g]) + for (auto sj : adj[si]) { + const auto h = groupOf[sj]; + if (h <= g) continue; + if (mixedBC[seeds.node[groups[h].front()]]) continue; + if (groups[g].size() + groups[h].size() > max_group) continue; + if (!sameSig(groups[h].front(), groups[g].front())) continue; bool seen = false; - for (auto& s : shared) - if (s.first == h) { - s.second++; + for (auto& t : shared) + if (t.first == h) { + t.second++; seen = true; break; } if (!seen) shared.emplace_back(h, 1); } + /*--- Shared adjacency is doubled so a junction-adjacent merge can be ranked above another of + * the same shape without ever outranking a better shape: a square scores 4 or 5 and a strip + * 2 or 3, so squares still come first and the junction only breaks ties among equals. ---*/ + const unsigned long boostG = groupNextToMixed(g); + for (const auto& t : shared) + merges.push_back( + {g, t.first, 2 * t.second + (boostG || groupNextToMixed(t.first) ? 1u : 0u), gkey[g], gkey[t.first]}); + } - auto bestH = std::numeric_limits::max(); - unsigned long bestShared = 0; - for (const auto& s : shared) - if (s.second > bestShared) { - bestShared = s.second; - bestH = s.first; - } + /*--- Best first, over ALL groups at once. Sweeping the groups in index order instead and letting + * each take its own best partner is what produced the strips: a group with no square partner + * left would take a weight-1 merge and consume a group that a later one needed for its square, + * and the failures cascade - on a structured 3D wall that came to 19% of the footprints. + * Ordering the merges globally means every square in the mesh is made before the first strip is + * even considered, so a strip only forms where the surface genuinely offers nothing better. ---*/ + std::sort(merges.begin(), merges.end(), [](const CMerge& a, const CMerge& b) { + if (a.weight != b.weight) return a.weight > b.weight; + if (a.keyG != b.keyG) return a.keyG < b.keyG; + return a.keyH < b.keyH; + }); + + consumed.assign(nGroups, 0); + vector> merged; + merged.reserve(nGroups); - if (bestH != std::numeric_limits::max()) { - consumed[bestH] = 1; - group.insert(group.end(), groups[bestH].begin(), groups[bestH].end()); - } + for (const auto& m : merges) { + if (consumed[m.g] || consumed[m.h]) continue; + consumed[m.g] = consumed[m.h] = 1; + auto group = groups[m.g]; + group.insert(group.end(), groups[m.h].begin(), groups[m.h].end()); merged.push_back(std::move(group)); } + /*--- Whatever found no partner passes through unchanged. ---*/ + for (unsigned long g = 0; g < nGroups; ++g) + if (!consumed[g]) merged.push_back(std::move(groups[g])); groups = std::move(merged); for (unsigned long g = 0; g < groups.size(); ++g) - for (auto li : groups[g]) groupOf[li] = g; + for (auto si : groups[g]) groupOf[si] = g; } return groups; @@ -1855,159 +2192,545 @@ vector> CMultiGridGeometry::BundleImplicitLines(const vect void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config) { - const auto starting_Index_CoarseCV = Index_CoarseCV; - - const auto stiff = ComputeNodeStiffness(fine_grid); - - /*--- PHASE A and B. Neither may be skipped on a rank with no lines: PHASE A takes a collective to - * agree on which markers carry a layer, and the summary at the end takes another. ---*/ - const auto lines = BuildImplicitLines(fine_grid, config, stiff); - - vector> adj, bundles; - if (!lines.empty()) bundles = BundleImplicitLines(lines, fine_grid, config, adj); - /*================================================================================================== - * PHASE C - extrude each bundle into a stack of coarse control volumes. + * Paving by advancing fronts. + * + * The boundary is agglomerated first into patches (PHASE 1), and every patch then rises into the + * domain as a FRONT that keeps its footprint: at each round every front node picks a successor, + * and the front advances only if all of them succeeded and none was lost to another front. So the + * layers of one stack are congruent by construction, and a coarse CV can never contain a node that + * belongs over a neighbouring patch. + * + * This is the part the earlier line-based version could not guarantee, and the reason it is gone. + * There, each wall node grew its own line first and the lines were grouped into bundles only + * afterwards, so nothing tied a line to the footprint it would later be asked to share: the + * marching direction was re-set to the last step taken every step, which let a line random-walk + * tangentially one legal 30-degree step at a time into a neighbouring column, and the connectivity + * test that should have caught it was applied to the lines' WALL roots, which stay adjacent no + * matter how far apart their tops drift. Ragged line lengths then made it worse, because a bundle + * was allowed to carry on with whichever subset of its lines was still long enough, so the stack + * changed footprint as it rose. + * + * Three rules replace all of that: * - * The bundle's boundary nodes become one coarse CV and each successive layer becomes the next, so - * the coarse grid inherits the layer structure of the fine grid and a line relaxation stays - * meaningful on it. PHASE A made the lines node-disjoint and PHASE B made the bundles a partition, - * so no two bundles can contend for a node and a stack is never interrupted part way up. + * - The front is the primitive. Nothing marches except a whole patch, so there is no such thing as + * an individual line to drift. + * - All-or-nothing layers. A front that cannot fill an entire layer retires and leaves the rest to + * ordinary agglomeration, instead of continuing narrower. + * - Contention resolved globally, once per layer, from bids collected before any is granted. Two + * fronts reaching for the same node is exactly the event "the fronts have met", and it is caught + * in the layer where it happens rather than fifteen layers later. * - * How many fine layers go into one coarse CV decides what is coarsened. One layer coarsens only - * tangentially and leaves the wall-normal line intact, which is what a line-implicit smoother needs - * where the cells are stretched; two layers coarsen in every direction, which is what the far field - * wants. MG_IMPLICIT_LINES_ISO_AR switches between them by the local aspect ratio as the stack - * rises, so the same stack can start semi-coarsened at the wall and finish isotropic - and because - * each semi-coarsening halves the stretching, the switch happens by itself at whatever level the - * mesh stops being anisotropic, rather than being tied to a multigrid level. With it at 0 the old - * behaviour stands and MG_IMPLICIT_LINES_ISOTROPIC decides for the whole stack. + * A front is stopped by TWO things and nothing else: reaching a boundary, or being unable to lay a + * layer topologically identical to the one it is standing on (see layerIsIsomorphic below). There + * is no limit on how far it may turn and no threshold on how stretched the mesh has to be. Those + * limits used to exist and they were the wrong instrument: an aspect-ratio cut in particular stops + * each front on a contour of the LOCAL cell shape, which on a flat plate is a contour of the + * streamwise spacing, so fronts died at different heights and the paved region ended in a + * staircase - and, being driven by dx rather than by the boundary layer, a staircase running the + * wrong way. Without it the same case paves to the far boundary at a uniform depth and the domain + * pass has nothing left to do. + * + * Each coarse CV is the front's footprint taken TWO layers deep, so the paving coarsens by the same + * factor in every direction: a 2x2 wall patch and the two layers above it make the 2x2x2 block, and + * the next CV of the stack starts from the footprint the front already has. Taking one layer per CV + * instead leaves the wall-normal direction uncoarsened entirely, which is what a line-implicit + * smoother wants and not what this is for. The single-layer CV survives in one place only: the top + * of a stack that retires with one layer buffered, where the alternative is dropping it back to + * ordinary agglomeration. * * The multigrid queue is deliberately not touched here: the sync loop after the boundary * agglomeration removes every point already marked agglomerated, so doing it here too would be an * error. *================================================================================================*/ - const bool ISOTROPIC = config->GetMGOptions().MG_Implicit_Lines_Isotropic; - const su2double ISO_AR = config->GetMGOptions().MG_Implicit_Lines_Iso_AR; - const bool HYBRID = (ISO_AR > 0.0); - - /*--- Are the still-growing lines one connected patch on the wall? A set that is no longer connected - * would put two separated columns into a single CV. Scans PHASE B's adjacency, over at most - * max_group entries, so the quadratic form is cheap. ---*/ - auto isConnected = [&adj](const vector& members, const vector& act) { - if (act.size() <= 1) return true; - vector seen(act.size(), 0); - vector stack{0}; - seen[0] = 1; - unsigned long nSeen = 1; - while (!stack.empty()) { - const auto cur = stack.back(); - stack.pop_back(); - const auto& neighbors = adj[members[act[cur]]]; - for (unsigned long k = 0; k < act.size(); ++k) { - if (seen[k]) continue; - if (find(neighbors.begin(), neighbors.end(), members[act[k]]) != neighbors.end()) { - seen[k] = 1; - nSeen++; - stack.push_back(k); - } - } + const auto starting_Index_CoarseCV = Index_CoarseCV; + const auto nPointFine = fine_grid->GetnPoint(); + const auto nMarkerFine = fine_grid->GetnMarker(); + constexpr auto NO_POINT = std::numeric_limits::max(); + const short int maxAgglomSize = (nDim == 2) ? 4 : 8; + + /*--- How nearly parallel a step must be to a boundary's normal to count as running INTO that + * boundary rather than along it. This is not a limit on where a front may go - it is only how + * "the front has reached a boundary" is recognised. ---*/ + constexpr su2double BOUNDARY_ALIGN_DEG = 30.0; + const su2double cos_boundary = cos(BOUNDARY_ALIGN_DEG * PI_NUMBER / 180.0); + /*--- Weight of the new step direction when the front's marching direction is updated. The direction + * only ever RANKS candidates, it never rejects one, so this is a preference and not a limit. ---*/ + constexpr su2double DIR_BLEND = 0.5; + + /*--- Safety cap on stack depth, 0 for none. A front is meant to run until it reaches a boundary or + * the mesh stops offering a clean extrusion, so this is off by default. ---*/ + const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; + unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; + if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; + + const auto stiff = ComputeNodeStiffness(fine_grid); + + /*--- PHASE 1. SeedFrontNodes must be reached by every rank, including one that owns no boundary: + * it takes a collective to agree on which markers carry a layer. ---*/ + const auto seeds = SeedFrontNodes(fine_grid, config, stiff); + const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); + const auto patches = BuildFrontPatches(seeds, fine_grid, config, mixedBC); + + /*--- Nodes on a boundary that carries a boundary condition. A front must not grow into one: those + * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two + * boundaries. CPoint's Boundary flag cannot answer this, as it is also set by SEND_RECEIVE, so on + * a partitioned mesh it is true for ordinary interior nodes of the send fringe and every front + * would stop one layer short of the partition. Walking the markers' own vertex lists is both + * exact and cheaper than testing every point against every marker. ---*/ + vector onPhysicalBoundary(nPointFine, 0); + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) + onPhysicalBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; + } + + /*--- A node on a physical boundary only ENDS a front if the front is stepping INTO that boundary, + * i.e. the step direction is roughly parallel to the boundary's own normal. A node that merely + * runs ALONG a boundary - a column on a spanwise symmetry plane, say - has that boundary's normal + * roughly PERPENDICULAR to the step, and is a legitimate interior node of the stack, not its end: + * onPhysicalBoundary alone cannot tell these apart, since it only records marker membership, not + * which direction the marker's surface runs in. Without this a front that happens to sit on a + * tangential boundary the whole way up dies at its very first step. ---*/ + auto entersBoundary = [&](unsigned long jPoint, const su2double* stepDir) { + for (unsigned short iMarker = 0; iMarker < nMarkerFine; iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + su2double n[MAXNDIM] = {0.0}; + if (!VertexUnitNormal(fine_grid, nDim, jPoint, iMarker, n)) continue; + if (fabs(GeometryToolbox::DotProduct(nDim, n, stepDir)) >= cos_boundary) return true; } - return nSeen == act.size(); + return false; + }; + + /*================================================================================================== + * PHASE 2 - advance every front, one layer per round. + *================================================================================================*/ + + /*--- One front node's chosen successor, before contention over it is resolved. ---*/ + struct CStep { + unsigned long node; /*!< Candidate successor. */ + unsigned long from; /*!< The front node that proposed it. */ + unsigned long key; /*!< Global index of "from", the final deterministic tie-break. */ + su2double score; /*!< Alignment of the step with the front's marching direction. */ + su2double dist; /*!< Length of the step. */ + su2double dir[MAXNDIM]; /*!< Unit step direction, reused to update the front's direction. */ }; - unsigned long nStacks = 0, nTruncated = 0, nSemiCV = 0, nFullCV = 0; + const auto nFront = patches.size(); + + vector> front(nFront), pending(nFront); + vector> dirNow(nFront); + vector alive(nFront, 0); + vector depth(nFront, 0), nBlock(nFront, 1), pendingLayers(nFront, 0); + vector> prop(nFront); + + vector claimed(nPointFine, 0); + /*--- Confirmed owner of a claimed node, -1 while free. Only ever written when a layer is accepted, + * so a bid that is still being contested never appears here. ---*/ + vector frontOf(nPointFine, -1); + + vector failed(nFront, 0); + vector failReason(nFront, 0); + vector stopCounts(N_STOP_REASONS, 0); + + /*--- The bid table. Only the index is kept per mesh point, and the bids themselves live in a + * compact vector holding one entry per candidate actually bid on this round - a few per front, + * against one entry per point in the mesh. Storing a whole CStep per point instead costs about + * sixty bytes times nPoint, which on the meshes this code is meant for is hundreds of megabytes + * of table that is empty almost everywhere. ---*/ + constexpr unsigned NOBID = std::numeric_limits::max(); + vector bidIdx(nPointFine, NOBID); + vector bids; + vector bidOwner; + + /*--- Scratch for the layer under construction, hoisted so a front does not allocate per layer. ---*/ + vector newLayer; + + unsigned long nStacks = 0, nSemiCV = 0, nFullCV = 0, nLayers = 0, nCovered = 0; unsigned long histogram[9] = {0}; - vector placed, active, group; - for (const auto& members : bundles) { - histogram[std::min(members.size(), 8)]++; + auto markFail = [&](unsigned long f, unsigned short why) { + if (!failed[f]) { + failed[f] = 1; + failReason[f] = why; + } + }; - /*--- The boundary CV. Claiming it before ordinary boundary agglomeration runs is what keeps the - * stack aligned: the layer above has exactly the same footprint. ---*/ - bool valid = true; - for (auto li : members) - if (!GeometricalCheck(lines[li][0], fine_grid, config)) valid = false; - if (!valid) continue; + /*--- How many fine layers the next coarse CV of this front holds: always two, so the stack coarsens + * by the same factor along the marching direction as the footprint does across it. The only + * exception is a footprint already so wide that a second layer would exceed the agglomeration + * size limit, which can only happen if MG_IMPLICIT_LINES_MAX_GROUP was raised past the + * dimension's default. ---*/ + auto blockFor = [&](const vector& layer) -> unsigned long { + return (layer.size() * 2 > static_cast(maxAgglomSize)) ? 1 : 2; + }; - for (unsigned long c = 0; c < members.size(); ++c) { - const auto p = lines[members[c]][0]; + /*--- Turn everything buffered for this front into one coarse control volume. ---*/ + auto emit = [&](unsigned long f) { + if (pending[f].empty()) return; + for (unsigned long c = 0; c < pending[f].size(); ++c) { + const auto p = pending[f][c]; fine_grid->nodes->SetParent_CV(p, Index_CoarseCV); nodes->SetChildren_CV(Index_CoarseCV, c, p); + if (fine_grid->nodes->GetAgglomerate_Indirect(p)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); } - nodes->SetnChildren_CV(Index_CoarseCV, static_cast(members.size())); + nodes->SetnChildren_CV(Index_CoarseCV, static_cast(pending[f].size())); Index_CoarseCV++; - nStacks++; + nCovered += pending[f].size(); + ((pendingLayers[f] == 1) ? nSemiCV : nFullCV)++; - /*--- The interior layers, in lockstep. A line that runs out stops contributing and the others - * carry on, so the stack narrows as it rises instead of being cut to its shortest member. - * Dropping below two lines would extrude a column one line wide, thinner than anything the - * domain pass would build there, so the stack ends instead; a bundle that only ever had one - * line is exempt, being one line wide by construction. ---*/ - const unsigned long minActive = std::min(2, members.size()); - placed.assign(members.size(), 1); - - auto collectActive = [&](unsigned long first, unsigned long blk) { - active.clear(); - for (unsigned long m = 0; m < members.size(); ++m) - if (first + blk <= lines[members[m]].size()) active.push_back(m); - }; + pending[f].clear(); + pendingLayers[f] = 0; + nBlock[f] = blockFor(front[f]); + }; + + auto isAdjacent = [&](unsigned long a, unsigned long b) { + const auto& pts = fine_grid->nodes->GetPoints(a); + return std::find(pts.begin(), pts.end(), b) != pts.end(); + }; + + /*--- The one test that decides whether a front may advance: is the layer it is about to lay down + * TOPOLOGICALLY IDENTICAL to the layer it is standing on? + * + * "old" and "new" are index-aligned, so phi maps old[k] to new[k], and phi is the extrusion the + * front is proposing. It is a valid layer exactly when phi is an isomorphism of the two induced + * subgraphs AND matches them up one for one: + * + * - same number of cells, which index alignment already gives; + * - every new cell is adjacent to exactly one old cell, and that one is its own preimage; + * - every old cell is adjacent to exactly one new cell, and that one is its own image; + * - the same edges: old[k]-old[l] is an edge if and only if new[k]-new[l] is, which makes the + * edge counts equal and carries connectivity across from the old layer for free. + * + * Nothing else stops a front. There is no cone on how far it may turn and no threshold on how + * stretched the mesh has to be: it runs until it reaches a boundary or until the mesh stops + * offering a clean extrusion, and this is what "stops offering" means. ---*/ + auto layerIsIsomorphic = [&](const vector& oldL, const vector& newL) { + const auto n = oldL.size(); + if (newL.size() != n) return false; + + for (size_t k = 0; k < n; ++k) { + unsigned nOld = 0, nNew = 0; + for (size_t l = 0; l < n; ++l) { + nOld += isAdjacent(newL[k], oldL[l]); + nNew += isAdjacent(oldL[k], newL[l]); + } + /*--- Exactly one partner each way, and it has to be the one phi names. ---*/ + if ((nOld != 1) || (nNew != 1)) return false; + if (!isAdjacent(oldL[k], newL[k])) return false; + } + + for (size_t k = 0; k < n; ++k) + for (size_t l = k + 1; l < n; ++l) + if (isAdjacent(oldL[k], oldL[l]) != isAdjacent(newL[k], newL[l])) return false; + + return true; + }; + + /*--- The boundary layer of every front. Claiming it before ordinary boundary agglomeration runs is + * what keeps the stack aligned: every layer above has exactly the same footprint. ---*/ + for (unsigned long f = 0; f < nFront; ++f) { + /*--- A one-node patch is allowed to seed a front and marches as a stack one node wide. It looks + * like a poor footprint, but the alternative - leaving it to ordinary agglomeration - is far + * worse: a seed that cannot pair is one whose whole COLUMN then goes unpaved, from the wall to + * wherever the front would have stopped, and those columns land in exactly the places that + * cannot pair for a reason. On the flat plate they were the two nodes at the leading edge, + * where the wall meets the symmetry plane and the marker signatures differ, plus the two + * domain corners: five full-height stripes cut through the paved region, the worst of them + * right at the leading edge. One-wide stacks leave no such hole, and since a coarse CV is now + * always two layers deep they still hold two nodes each rather than one. ---*/ + bool valid = !patches[f].empty(); + for (auto si : patches[f]) { + const auto p = seeds.node[si]; + if (!GeometricalCheck(p, fine_grid, config) || fine_grid->nodes->GetAgglomerate(p)) valid = false; + } + if (!valid) continue; - for (unsigned long first = 1;;) { - unsigned long nBlock = ISOTROPIC ? 2 : 1; - if (HYBRID) { - /*--- Stay semi-coarsened while any line of this bundle is still in stretched mesh at this - * height; the wall-normal direction is shared by the whole stack. ---*/ - su2double arHere = 0.0; - for (unsigned long m = 0; m < members.size(); ++m) - if (first < lines[members[m]].size()) - arHere = std::max(arHere, stiff.AspectRatio(lines[members[m]][first])); - nBlock = (arHere > ISO_AR) ? 1 : 2; + std::array n0{}; + for (auto si : patches[f]) { + front[f].push_back(seeds.node[si]); + claimed[seeds.node[si]] = 1; + frontOf[seeds.node[si]] = static_cast(f); + for (unsigned short d = 0; d < nDim; ++d) n0[d] += seeds.normal[si][d]; + } + const su2double nrm = GeometryToolbox::Norm(nDim, n0.data()); + if (nrm <= 0.0) { + /*--- A patch whose members' normals cancel has no direction to march in. ---*/ + for (auto p : front[f]) { + claimed[p] = 0; + frontOf[p] = -1; } + front[f].clear(); + continue; + } + for (unsigned short d = 0; d < nDim; ++d) n0[d] /= nrm; + dirNow[f] = n0; + + histogram[std::min(front[f].size(), 8)]++; + alive[f] = 1; + nStacks++; + nLayers++; + + pending[f] = front[f]; + pendingLayers[f] = 1; + /*--- A front rooted on a node where two different boundary conditions meet emits that node as a + * coarse CV of its own, one fine node deep, so the junction is never averaged into a control + * volume with anything else. Only the FIRST CV of the stack is treated this way: emit() then + * asks blockFor again for what is by then an ordinary interior layer, and the rest of the + * stack rises two nodes at a time like any other. ---*/ + nBlock[f] = mixedBC[front[f].front()] ? 1 : blockFor(front[f]); + if (pendingLayers[f] >= nBlock[f]) emit(f); + } - collectActive(first, nBlock); - /*--- One layer short of a full block at the top, take it as a single layer rather than drop it. ---*/ - if ((nBlock == 2) && (active.size() < minActive)) { - nBlock = 1; - collectActive(first, nBlock); + for (unsigned long layer = 1;; ++layer) { + bool anyAlive = false; + for (unsigned long f = 0; f < nFront; ++f) anyAlive = anyAlive || alive[f]; + if (!anyAlive) break; + + std::fill(failed.begin(), failed.end(), 0); + for (const auto& b : bids) bidIdx[b.node] = NOBID; + bids.clear(); + bidOwner.clear(); + + /*--- (a) Every alive front proposes a successor for each of its nodes. A front that cannot fill a + * whole layer proposes NOTHING: it is retiring this round anyway, and letting its partial bids + * stand would let a dying front displace a healthy one out of nodes it can still use. ---*/ + for (unsigned long f = 0; f < nFront; ++f) { + if (!alive[f]) continue; + prop[f].clear(); + + if ((MAX_LINE_LENGTH > 0) && (depth[f] + 1 >= MAX_LINE_LENGTH)) { + markFail(f, STOP_MAX_LENGTH); + continue; } - if (active.size() < minActive) break; - if (!isConnected(members, active)) break; + for (auto n : front[f]) { + auto best = NO_POINT; + su2double best_dot = -2.0, best_len = 0.0, best_dir[MAXNDIM] = {0.0}; + bool sawCollision = false, sawBoundary = false, sawAgglom = false, sawPartition = false; + bool sawGeom = false; + + for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(n); ++iNeigh) { + const auto jPoint = fine_grid->nodes->GetPoint(n, iNeigh); + + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(n), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) continue; + for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; + + /*--- The marching direction RANKS the candidates and nothing more: whichever free neighbour + * lies most nearly ahead is the one proposed. There is no cone, so a front is never + * stopped for turning - only for running out of mesh to extrude into. ---*/ + const su2double dot = GeometryToolbox::DotProduct(nDim, vec, dirNow[f].data()); + + /*--- Halo nodes stay out: their parent is dictated by the rank that owns them and arrives + * through the MPI relay, so a front claiming one would fight that assignment. This is + * what a front hits when it reaches a partition interface, and it needs a reason of its + * own, rather than being skipped silently and leaving some other candidate to explain a + * stop that was really the partitioning. ---*/ + if (!fine_grid->nodes->GetDomain(jPoint)) { + sawPartition = true; + continue; + } + if (fine_grid->nodes->GetAgglomerate(jPoint)) { + /*--- A node another front has already turned into a coarse CV also reads as agglomerated, + * so ask frontOf first: that is the fronts meeting, not an earlier phase. ---*/ + if ((frontOf[jPoint] >= 0) && (frontOf[jPoint] != static_cast(f))) + sawCollision = true; + else if (frontOf[jPoint] < 0) + sawAgglom = true; + continue; + } + if (claimed[jPoint]) { + if (frontOf[jPoint] != static_cast(f)) sawCollision = true; + continue; + } - group.clear(); - group.reserve(active.size() * nBlock); - for (auto m : active) - for (unsigned long b = 0; b < nBlock; ++b) group.push_back(lines[members[m]][first + b]); + if (onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) { + sawBoundary = true; + continue; + } + if (!GeometricalCheck(jPoint, fine_grid, config)) { + sawGeom = true; + continue; + } - valid = true; - for (auto p : group) - if (!GeometricalCheck(p, fine_grid, config)) valid = false; - if (!valid) break; + if (dot > best_dot) { + best_dot = dot; + best = jPoint; + best_len = len; + for (unsigned short d = 0; d < nDim; ++d) best_dir[d] = vec[d]; + } + } - for (unsigned long c = 0; c < group.size(); ++c) { - fine_grid->nodes->SetParent_CV(group[c], Index_CoarseCV); - nodes->SetChildren_CV(Index_CoarseCV, c, group[c]); + if (best == NO_POINT) { + /*--- Priority order picks the cleanest explanation first: reaching a physical boundary is a + * correct, expected stop and takes priority even if some other, non-viable candidate also + * happened to be claimed. Only report a collision when no boundary was involved. ---*/ + if (sawBoundary) + markFail(f, STOP_PHYS_BOUNDARY); + else if (sawPartition) + markFail(f, STOP_PARTITION); + else if (sawCollision) + markFail(f, STOP_COLLISION); + else if (sawAgglom) + markFail(f, STOP_AGGLOMERATED); + else if (sawGeom) + markFail(f, STOP_GEOMETRY); + else + markFail(f, STOP_NO_NEIGHBOR); + break; + } + + CStep s{}; + s.node = best; + s.from = n; + s.key = fine_grid->nodes->GetGlobalIndex(n); + s.score = best_dot; + s.dist = best_len; + for (unsigned short d = 0; d < nDim; ++d) s.dir[d] = best_dir[d]; + prop[f].push_back(s); } - nodes->SetnChildren_CV(Index_CoarseCV, static_cast(group.size())); - Index_CoarseCV++; - ((nBlock == 1) ? nSemiCV : nFullCV)++; - for (auto m : active) placed[m] = first + nBlock; - first += nBlock; + if (failed[f]) prop[f].clear(); } - for (unsigned long m = 0; m < members.size(); ++m) nTruncated += lines[members[m]].size() - placed[m]; - } + /*--- (b) Contention resolved from bids that were all collected before any was granted, so the + * outcome is a pure function of the proposals and does not depend on the order the fronts are + * visited in. That is what makes the coarse grid reproducible. ---*/ + auto better = [](const CStep& a, const CStep& b) { + if (a.score != b.score) return a.score > b.score; + if (a.dist != b.dist) return a.dist < b.dist; + return a.key < b.key; + }; - /*--- Summary over all ranks. Reporting rank 0's own lines, as this used to, makes a partitioned run - * look like a fraction of the mesh it is not, and hides how much of the layer the partitioning - * cost: a line stops at the partition, so the count of nodes left to ordinary agglomeration is - * the number to watch when adding ranks. Every rank must reach these collectives. ---*/ - unsigned long nLineNodes = 0; - for (const auto& L : lines) nLineNodes += L.size(); + for (unsigned long f = 0; f < nFront; ++f) { + if (!alive[f] || failed[f]) continue; + for (const auto& s : prop[f]) { + /*--- A front that has just lost a bid is retiring, so it must not go on to place the rest and + * displace a front that is still healthy. What it placed BEFORE losing does stay in the + * table, and can still cost another front a candidate it would otherwise have won: the + * only way to avoid that entirely is to re-run the contention to a fixed point after every + * retirement. The residual is conservative - it retires a front near a seam one layer + * early, never merges anything it should not - and the seam goes to ordinary agglomeration + * either way, so it is not worth an inner iteration. ---*/ + if (failed[f]) break; + + if (bidIdx[s.node] == NOBID) { + bidIdx[s.node] = static_cast(bids.size()); + bids.push_back(s); + bidOwner.push_back(f); + continue; + } - unsigned long local[6] = {lines.size(), nStacks, nLineNodes, nTruncated, nSemiCV, nFullCV}; + const auto k = bidIdx[s.node]; + const auto g = bidOwner[k]; + /*--- Two nodes of the SAME front reaching for one successor is a pinch: the layer would come + * out narrower than the front, which all-or-nothing does not allow. ---*/ + const unsigned short why = (g == f) ? STOP_PINCH : STOP_COLLISION; + + if (better(s, bids[k])) { + markFail(g, why); + bids[k] = s; + bidOwner[k] = f; + } else { + markFail(f, why); + } + + /*--- A head-on meeting stops BOTH fronts. Letting the winner carry on through the seam would + * push its stack into territory the other front had every right to, and the asymmetry + * shows up in the coarse grid as one stack overshooting the other. A glancing contact + * (directions not opposed) is not a meeting and only costs the loser. ---*/ + if ((g != f) && (GeometryToolbox::DotProduct(nDim, dirNow[f].data(), dirNow[g].data()) < 0.0)) { + markFail(f, STOP_COLLISION); + markFail(g, STOP_COLLISION); + } + } + } + + /*--- (c) All-or-nothing acceptance: a front takes the whole layer or none of it and retires. A + * bid only becomes a claim here, so a retiring front never has to give anything back and the + * nodes it was reaching for stay available to ordinary agglomeration. ---*/ + for (unsigned long f = 0; f < nFront; ++f) { + if (!alive[f]) continue; + + newLayer.clear(); + if (!failed[f]) { + /*--- Built in the order prop[f] was, which is the order of front[f], so newLayer[k] is the + * successor proposed by front[f][k] and the two vectors carry phi between them. ---*/ + for (const auto& s : prop[f]) { + const auto k = bidIdx[s.node]; + if ((k != NOBID) && (bidOwner[k] == f)) newLayer.push_back(s.node); + } + /*--- Every bid of a front that was not marked failed must have been granted. ---*/ + if (newLayer.size() != front[f].size()) + markFail(f, STOP_COLLISION); + else if (!layerIsIsomorphic(front[f], newLayer)) + markFail(f, STOP_TOPOLOGY); + } + + if (failed[f]) { + /*--- Nothing to give back: a bid only becomes a claim on acceptance below. ---*/ + alive[f] = 0; + stopCounts[failReason[f]]++; + /*--- One layer short of a full block at the top: take what is buffered as its own coarse CV + * rather than dropping it back to ordinary agglomeration. ---*/ + emit(f); + continue; + } + + /*--- Accept. The direction is blended rather than replaced, and it is the FRONT's direction, + * updated once from the mean of the steps its nodes just took, not one direction per node + * free to wander off on its own. ---*/ + su2double mean[MAXNDIM] = {0.0}; + for (const auto& s : prop[f]) + for (unsigned short d = 0; d < nDim; ++d) mean[d] += s.dir[d]; + const su2double meanNrm = GeometryToolbox::Norm(nDim, mean); + if (meanNrm > 0.0) { + su2double blended[MAXNDIM] = {0.0}; + for (unsigned short d = 0; d < nDim; ++d) + blended[d] = (1.0 - DIR_BLEND) * dirNow[f][d] + DIR_BLEND * mean[d] / meanNrm; + const su2double bNrm = GeometryToolbox::Norm(nDim, blended); + if (bNrm > 0.0) + for (unsigned short d = 0; d < nDim; ++d) dirNow[f][d] = blended[d] / bNrm; + } + + for (auto p : newLayer) { + claimed[p] = 1; + frontOf[p] = static_cast(f); + } + front[f] = std::move(newLayer); + depth[f]++; + nLayers++; + + pending[f].insert(pending[f].end(), front[f].begin(), front[f].end()); + pendingLayers[f]++; + if (pendingLayers[f] >= nBlock[f]) emit(f); + } + } + + /*--- Nothing should be left buffered, but a front retired outside the loop would strand its nodes + * with a parent index that was never assigned. ---*/ + for (unsigned long f = 0; f < nFront; ++f) emit(f); + + /*--- How far each front actually got. This is the number to watch when the paved region does not + * look like a front: fronts that all reach the same height leave a flat interface with ordinary + * agglomeration, and a spread here is that interface coming out as a staircase instead. ---*/ + unsigned long dmin = std::numeric_limits::max(), dmax = 0; + for (unsigned long f = 0; f < nFront; ++f) { + if (front[f].empty()) continue; + dmin = std::min(dmin, depth[f]); + dmax = std::max(dmax, depth[f]); + } + if (dmin == std::numeric_limits::max()) dmin = 0; + + /*--- Summary over all ranks. Reporting rank 0's own fronts makes a partitioned run look like a + * fraction of the mesh it is not, and hides how much of the layer the partitioning cost: a front + * stops at the partition, so the number of nodes left to ordinary agglomeration is the number to + * watch when adding ranks. Every rank must reach these collectives. ---*/ + unsigned long nSeedNodes = seeds.node.size(); + unsigned long local[6] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV}; unsigned long total[6] = {0}; SU2_MPI::Allreduce(local, total, 6, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -2017,15 +2740,32 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long histTotal[9] = {0}; SU2_MPI::Allreduce(histogram, histTotal, 9, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + vector stopTotal(N_STOP_REASONS, 0); + SU2_MPI::Allreduce(stopCounts.data(), stopTotal.data(), N_STOP_REASONS, MPI_UNSIGNED_LONG, MPI_SUM, + SU2_MPI::GetComm()); + + unsigned long depthMin = 0, depthMax = 0; + SU2_MPI::Allreduce(&dmin, &depthMin, 1, MPI_UNSIGNED_LONG, MPI_MIN, SU2_MPI::GetComm()); + SU2_MPI::Allreduce(&dmax, &depthMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); + if (rank == MASTER_NODE) { - cout << " Implicit lines: " << total[0] << " lines, " << total[1] << " stacks, bundle sizes "; + cout << " Paving fronts: " << total[1] << " fronts from " << total[0] << " seed nodes, patch sizes "; for (unsigned s = 1; s <= 8; ++s) if (histTotal[s] > 0) cout << s << "x" << histTotal[s] << " "; - cout << "\n Coarse CVs from lines: " << totalCV << " covering " << (total[2] - total[3]) << "/" << total[2] - << " line nodes"; - if (total[3] > 0) cout << " (" << total[3] << " left to domain agglomeration)"; + cout << "\n Coarse CVs from fronts: " << totalCV << " covering " << total[3] << " nodes in " << total[2] + << " layers, front depth " << depthMin << " to " << depthMax; if (total[4] + total[5] > 0) - cout << "\n Stack layers: " << total[4] << " semi-coarsened, " << total[5] << " isotropic"; + cout << "\n Coarse CVs by depth: " << total[5] << " two layers deep, " << total[4] + << " one layer (isolated junction, or top of a stack)"; + /*--- Why each front stopped. Reaching a boundary is the one correct stop; everything else is the + * mesh failing to offer a layer topologically identical to the current one, broken down by how + * it failed. COLLISION and PINCH are two fronts, or two nodes of one front, reaching for the + * same node; TOPOLOGY is a layer that was claimable but not an extrusion. ---*/ + cout << "\n Front advance stopped due to: physical-boundary " << stopTotal[STOP_PHYS_BOUNDARY] << ", partition " + << stopTotal[STOP_PARTITION] << ", front-collision " << stopTotal[STOP_COLLISION] << ", pinch " + << stopTotal[STOP_PINCH] << ", already-agglomerated " << stopTotal[STOP_AGGLOMERATED] << ", dead-end " + << stopTotal[STOP_NO_NEIGHBOR] << ", topology " << stopTotal[STOP_TOPOLOGY] << ", geometry " + << stopTotal[STOP_GEOMETRY] << ", max-length " << stopTotal[STOP_MAX_LENGTH]; cout << endl; } } diff --git a/Common/src/geometry/CPhysicalGeometry.cpp b/Common/src/geometry/CPhysicalGeometry.cpp index 38bf0ebea8a..e61df306c34 100644 --- a/Common/src/geometry/CPhysicalGeometry.cpp +++ b/Common/src/geometry/CPhysicalGeometry.cpp @@ -7432,22 +7432,32 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { vector adjwgt; const su2double anisoWgt = config->GetParMETIS_AnisoWeight(); - - if (anisoWgt > 0.0) { - const auto firstIdx = pointPartitioner.GetFirstIndexOnRank(rank); - const auto lastIdx = pointPartitioner.GetLastIndexOnRank(rank); - auto isLocal = [&](unsigned long g) { return g >= firstIdx && g < lastIdx; }; - + const bool columnPart = config->GetParMETIS_ColumnPartition(); + + const auto firstIdx = pointPartitioner.GetFirstIndexOnRank(rank); + const auto lastIdx = pointPartitioner.GetLastIndexOnRank(rank); + auto isLocal = [&](unsigned long g) { return g >= firstIdx && g < lastIdx; }; + + /*--- Edge lengths and the longest edge at each point. Shared by the edge weights below and by the + * column contraction, both of which need to know how a given edge compares with the ones around + * it. Only the coordinates are available at this point of the setup, the dual grid is built much + * later, so length stands in for the face area over volume ratio used elsewhere. ---*/ + vector edgeLen, maxLen, minLen; + vector isColEdge; + vector> stiffDir; + vector nSend(size, 0), nRecv(size, 0), sDisp(size + 1, 0), rDisp(size + 1, 0); + vector sendIdx, recvIdx; + map remoteMaxLen; + + if ((anisoWgt > 0.0) || columnPart) { /*--- The graph is split linearly and its entries are global indices, so an edge near a linear * partition boundary has one end that is not stored here. Those are few, of the order of a * percent of the entries, and each is asked for from the rank the linear partitioner says - * owns it. The same request lists are used twice, once for coordinates and once for the - * longest edge at the point, which only its owner can work out. ---*/ + * owns it. The same request lists are reused for every later exchange. ---*/ vector> wanted(size); for (auto gPoint : adjacency) if (!isLocal(gPoint)) wanted[pointPartitioner.GetRankContainingIndex(gPoint)].push_back(gPoint); - vector nSend(size, 0), nRecv(size, 0), sDisp(size + 1, 0), rDisp(size + 1, 0); for (int r = 0; r < size; ++r) { auto& w = wanted[r]; sort(w.begin(), w.end()); @@ -7460,7 +7470,8 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { rDisp[r + 1] = rDisp[r] + nRecv[r]; } - vector sendIdx(sDisp[size]), recvIdx(rDisp[size]); + sendIdx.resize(sDisp[size]); + recvIdx.resize(rDisp[size]); for (int r = 0; r < size; ++r) copy(wanted[r].begin(), wanted[r].end(), sendIdx.begin() + sDisp[r]); SU2_MPI::Alltoallv(sendIdx.data(), nSend.data(), sDisp.data(), MPI_UNSIGNED_LONG, recvIdx.data(), nRecv.data(), rDisp.data(), MPI_UNSIGNED_LONG, comm); @@ -7490,10 +7501,10 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { } } - /*--- Length of every edge of the local part of the graph, and the longest edge at each point - * this rank owns. ---*/ - vector edgeLen(adjacency.size(), 0.0); - vector maxLen(nPoint, 0.0); + edgeLen.assign(adjacency.size(), 0.0); + maxLen.assign(nPoint, 0.0); + minLen.assign(nPoint, std::numeric_limits::max()); + stiffDir.assign(nPoint, {0.0, 0.0, 0.0}); for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { @@ -7510,11 +7521,18 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { } edgeLen[k] = GeometryToolbox::Distance(nDim, nodes->GetCoord(iPoint), coord_j); maxLen[iPoint] = max(maxLen[iPoint], edgeLen[k]); + + /*--- The shortest edge at a node fixes the direction its column runs in. ---*/ + if (edgeLen[k] < minLen[iPoint]) { + minLen[iPoint] = edgeLen[k]; + for (unsigned short iDim = 0; iDim < nDim; ++iDim) + stiffDir[iPoint][iDim] = (coord_j[iDim] - nodes->GetCoord(iPoint, iDim)) / edgeLen[k]; + } } + if (minLen[iPoint] == std::numeric_limits::max()) minLen[iPoint] = 0.0; } /*--- Round two, the longest edge at each of the remote points. ---*/ - map remoteMaxLen; { vector sendBuf(rDisp[size]), recvBuf(sDisp[size]); for (size_t i = 0; i < recvIdx.size(); ++i) sendBuf[i] = maxLen[recvIdx[i] - firstIdx]; @@ -7523,15 +7541,102 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { for (size_t i = 0; i < sendIdx.size(); ++i) remoteMaxLen[sendIdx[i]] = recvBuf[i]; } - /*--- An edge is expensive to cut when it is much shorter than the other edges meeting it, which - * is the definition of the local cell aspect ratio and is exactly the situation inside a - * boundary layer, where the short edges are the wall-normal ones. Comparing an edge only - * against its own neighbourhood, rather than against a global length, is what keeps the - * measure a ratio: scaling the whole mesh, or refining one region of it isotropically, leaves - * every weight unchanged. Weighting by absolute length instead would make any small cell - * expensive to cut and would steer the partitioner away from refined regions that are not - * stretched at all. Averaging the two ends keeps the weight of an edge symmetric, which - * ParMETIS requires. ---*/ + /*--- Which edges make up the wall-normal columns. + * + * An edge qualifies when it is short compared with the longest edge at both of its ends, so + * the mesh is stretched there, AND it runs along the shortest edge at both of its ends. The + * second test is what keeps a column one-dimensional: in a cell graded in two directions at + * once, several directions are shorter than the longest one and testing only the length links + * the columns sideways into sheets, which then contract into a handful of enormous vertices + * instead of one per wall node. Requiring alignment with the stiffest direction of both ends + * leaves exactly the chains that run through the layer. ---*/ + if (columnPart) { + map> remoteDir; + { + vector sendBuf(static_cast(rDisp[size]) * nDim), + recvBuf(static_cast(sDisp[size]) * nDim); + for (size_t i = 0; i < recvIdx.size(); ++i) + for (unsigned short iDim = 0; iDim < nDim; ++iDim) + sendBuf[i * nDim + iDim] = stiffDir[recvIdx[i] - firstIdx][iDim]; + vector nS(size), nR(size), sD(size), rD(size); + for (int r = 0; r < size; ++r) { + nS[r] = nRecv[r] * nDim; + nR[r] = nSend[r] * nDim; + sD[r] = rDisp[r] * nDim; + rD[r] = sDisp[r] * nDim; + } + SU2_MPI::Alltoallv(sendBuf.data(), nS.data(), sD.data(), MPI_DOUBLE, recvBuf.data(), nR.data(), rD.data(), + MPI_DOUBLE, comm); + for (size_t i = 0; i < sendIdx.size(); ++i) { + array d = {0.0, 0.0, 0.0}; + for (unsigned short iDim = 0; iDim < nDim; ++iDim) d[iDim] = recvBuf[i * nDim + iDim]; + remoteDir[sendIdx[i]] = d; + } + } + + const su2double COLUMN_FRACTION = 0.5; /*!< Stretched enough for the edge to be part of a column. */ + const su2double COLUMN_COS = 0.9; /*!< Aligned enough with the stiffest direction at both ends. */ + + isColEdge.assign(adjacency.size(), 0); + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { + if (edgeLen[k] <= 0.0) continue; + const auto gPoint = adjacency[k]; + + su2double maxLen_j = 0.0; + array dir_j = {0.0, 0.0, 0.0}, coord_j = {0.0, 0.0, 0.0}; + if (isLocal(gPoint)) { + maxLen_j = maxLen[gPoint - firstIdx]; + dir_j = stiffDir[gPoint - firstIdx]; + for (unsigned short iDim = 0; iDim < nDim; ++iDim) coord_j[iDim] = nodes->GetCoord(gPoint - firstIdx, iDim); + } else { + const auto itM = remoteMaxLen.find(gPoint); + const auto itD = remoteDir.find(gPoint); + const auto itC = remoteCoord.find(gPoint); + if ((itM == remoteMaxLen.end()) || (itD == remoteDir.end()) || (itC == remoteCoord.end())) continue; + maxLen_j = itM->second; + dir_j = itD->second; + coord_j = itC->second; + } + + if (edgeLen[k] > COLUMN_FRACTION * maxLen[iPoint]) continue; + if (edgeLen[k] > COLUMN_FRACTION * maxLen_j) continue; + + su2double e[3] = {0.0, 0.0, 0.0}, di = 0.0, dj = 0.0; + for (unsigned short iDim = 0; iDim < nDim; ++iDim) { + e[iDim] = (coord_j[iDim] - nodes->GetCoord(iPoint, iDim)) / edgeLen[k]; + di += e[iDim] * stiffDir[iPoint][iDim]; + dj += e[iDim] * dir_j[iDim]; + } + if ((fabs(di) < COLUMN_COS) || (fabs(dj) < COLUMN_COS)) continue; + + isColEdge[k] = 1; + } + } + } + } + + /*--- Cost of cutting each edge of the graph. + * + * Without these ParMETIS is given no edge weights at all and every edge is equally cheap to + * cut, so nothing stops a partition boundary from running straight through the stretched cells + * of a boundary layer and splitting the wall-normal columns that implicit-line agglomeration + * and line-implicit smoothing are built on. + * + * An edge is expensive to cut when it is much shorter than the other edges meeting it, which + * is the definition of the local cell aspect ratio and is exactly the situation inside a + * boundary layer, where the short edges are the wall-normal ones. Comparing an edge only + * against its own neighbourhood, rather than against a global length, is what keeps the + * measure a ratio: scaling the whole mesh, or refining one region of it isotropically, leaves + * every weight unchanged. Averaging the two ends keeps the weight of an edge symmetric, which + * ParMETIS requires. + * + * Note this only discourages such cuts, it cannot forbid them, and by making the wall-normal + * direction expensive it pushes every cut into the wall surface instead, which splits the wall + * faces that tangential bundling needs. PARMETIS_COLUMN_PARTITION below removes the choice + * rather than reweighting it. ---*/ + + if (anisoWgt > 0.0) { const idx_t MAX_EDGE_WEIGHT = 1000; adjwgt.resize(adjacency.size(), 1); @@ -7559,16 +7664,300 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { idx_t edgecut; vector part(nPoint); + bool coloured = false; + + /*================================================================================================== + * Column-contracted partitioning. + * + * Weighting edges can only make a cut across a boundary-layer column unattractive; the partitioner + * is still free to make it, and by pricing the wall-normal direction out it takes its cuts through + * the wall surface instead, which splits the wall faces that the tangential bundling of implicit + * lines is built from. Contracting removes the choice: each column of stretched cells becomes one + * vertex of the graph handed to ParMETIS, so no cut can pass through a column, and the graph that + * is actually cut is the wall surface with the layer hanging off it. + * + * A column is a connected component of the edges that are short at BOTH ends. At a wall node of a + * stretched cell the wall-normal edge is a fraction of the tangential ones, so it qualifies while + * the tangential ones do not, and the components come out as the wall-normal chains. Where the + * mesh is isotropic every edge is about as long as its neighbours, nothing qualifies, each point + * is its own column, and the contracted graph is the original one - such a mesh is partitioned + * exactly as it would have been. Nothing here needs to know where the walls are. + *================================================================================================*/ + + if (columnPart) { + constexpr unsigned long MAX_SWEEPS = 500; + + /*--- Label propagation: every point starts as its own column and repeatedly takes the smallest + * label across a column edge, so a column ends up named after its lowest global index. The + * number of sweeps needed is the length of the longest column, not the size of the mesh. ---*/ + vector label(nPoint); + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) label[iPoint] = firstIdx + iPoint; + + map remoteLabel; + for (auto g : sendIdx) remoteLabel[g] = g; + + unsigned long nSweeps = 0; + for (int changed = 1; changed != 0;) { + changed = 0; + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { + if (!isColEdge[k]) continue; + const auto gPoint = adjacency[k]; + unsigned long lj; + if (isLocal(gPoint)) { + lj = label[gPoint - firstIdx]; + } else { + const auto it = remoteLabel.find(gPoint); + if (it == remoteLabel.end()) continue; + lj = it->second; + } + if (lj < label[iPoint]) { + label[iPoint] = lj; + changed = 1; + } + } + } + + /*--- Refresh the labels of the points that are not stored here. ---*/ + { + vector sendBuf(rDisp[size]), recvBuf(sDisp[size]); + for (size_t i = 0; i < recvIdx.size(); ++i) sendBuf[i] = label[recvIdx[i] - firstIdx]; + SU2_MPI::Alltoallv(sendBuf.data(), nRecv.data(), rDisp.data(), MPI_UNSIGNED_LONG, recvBuf.data(), nSend.data(), + sDisp.data(), MPI_UNSIGNED_LONG, comm); + for (size_t i = 0; i < sendIdx.size(); ++i) { + if (recvBuf[i] != remoteLabel[sendIdx[i]]) changed = 1; + remoteLabel[sendIdx[i]] = recvBuf[i]; + } + } + + int global_changed = 0; + SU2_MPI::Allreduce(&changed, &global_changed, 1, MPI_INT, MPI_MAX, comm); + changed = global_changed; + + if (++nSweeps >= MAX_SWEEPS) break; + } + + /*--- A column is owned by whichever rank the linear partitioner says holds the index it is named + * after, which is one of its own points, so every column has exactly one owner. Number the + * columns consecutively, owner by owner, to give ParMETIS the contiguous distribution it + * expects. ---*/ + unsigned long nColLocal = 0; + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) + if (label[iPoint] == firstIdx + iPoint) nColLocal++; + + vector colCount(size, 0); + SU2_MPI::Allgather(&nColLocal, 1, MPI_UNSIGNED_LONG, colCount.data(), 1, MPI_UNSIGNED_LONG, comm); + + vector cvtxdist(size + 1, 0); + for (int r = 0; r < size; ++r) cvtxdist[r + 1] = cvtxdist[r] + static_cast(colCount[r]); + const auto nColGlobal = static_cast(cvtxdist[size]); + + map colIndexOfRoot; /*!< Global point index of a root -> column index. */ + { + unsigned long next = static_cast(cvtxdist[rank]); + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) + if (label[iPoint] == firstIdx + iPoint) colIndexOfRoot[firstIdx + iPoint] = next++; + } + + /*--- Ask the owner of each label for the column index it was given. ---*/ + auto askOwners = [&](const vector& keys, vector& values) { + vector> want(size); + for (auto key : keys) want[pointPartitioner.GetRankContainingIndex(key)].push_back(key); + vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); + for (int r = 0; r < size; ++r) { + sort(want[r].begin(), want[r].end()); + want[r].erase(unique(want[r].begin(), want[r].end()), want[r].end()); + nS[r] = static_cast(want[r].size()); + } + SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); + for (int r = 0; r < size; ++r) { + sD[r + 1] = sD[r] + nS[r]; + rD[r + 1] = rD[r] + nR[r]; + } + vector qs(sD[size]), qr(rD[size]); + for (int r = 0; r < size; ++r) copy(want[r].begin(), want[r].end(), qs.begin() + sD[r]); + SU2_MPI::Alltoallv(qs.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, qr.data(), nR.data(), rD.data(), + MPI_UNSIGNED_LONG, comm); + + vector as(rD[size]), ar(sD[size]); + for (size_t i = 0; i < qr.size(); ++i) { + const auto it = colIndexOfRoot.find(qr[i]); + as[i] = (it == colIndexOfRoot.end()) ? nColGlobal : it->second; + } + SU2_MPI::Alltoallv(as.data(), nR.data(), rD.data(), MPI_UNSIGNED_LONG, ar.data(), nS.data(), sD.data(), + MPI_UNSIGNED_LONG, comm); + + map answer; + for (int r = 0; r < size; ++r) + for (int i = sD[r]; i < sD[r + 1]; ++i) answer[qs[i]] = ar[i]; + + values.resize(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + const auto it = answer.find(keys[i]); + values[i] = (it == answer.end()) ? nColGlobal : it->second; + } + }; + + /*--- Column index of every point stored here, and of every point across a linear boundary. ---*/ + vector myKeys(nPoint); + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) myKeys[iPoint] = label[iPoint]; + vector colOfPoint; + askOwners(myKeys, colOfPoint); + + vector remoteKeys; + remoteKeys.reserve(sendIdx.size()); + for (auto g : sendIdx) remoteKeys.push_back(remoteLabel[g]); + vector colOfRemoteVal; + askOwners(remoteKeys, colOfRemoteVal); + map colOfRemote; + for (size_t i = 0; i < sendIdx.size(); ++i) colOfRemote[sendIdx[i]] = colOfRemoteVal[i]; + + /*--- Send every contracted edge, and every point's weight, to the rank owning the column. ---*/ + vector> outEdge(size), outWeight(size); + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + const auto cA = colOfPoint[iPoint]; + if (cA >= nColGlobal) continue; + int owner = 0; + while ((owner + 1 < size) && (static_cast(cA) >= cvtxdist[owner + 1])) owner++; + outWeight[owner].push_back(cA); + + for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { + const auto gPoint = adjacency[k]; + unsigned long cB; + if (isLocal(gPoint)) { + cB = colOfPoint[gPoint - firstIdx]; + } else { + const auto it = colOfRemote.find(gPoint); + if (it == colOfRemote.end()) continue; + cB = it->second; + } + if ((cB >= nColGlobal) || (cB == cA)) continue; + outEdge[owner].push_back(cA); + outEdge[owner].push_back(cB); + } + } + + auto shipPairs = [&](vector>& out, vector& in) { + vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); + for (int r = 0; r < size; ++r) nS[r] = static_cast(out[r].size()); + SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); + for (int r = 0; r < size; ++r) { + sD[r + 1] = sD[r] + nS[r]; + rD[r + 1] = rD[r] + nR[r]; + } + vector sb(sD[size]); + in.resize(rD[size]); + for (int r = 0; r < size; ++r) copy(out[r].begin(), out[r].end(), sb.begin() + sD[r]); + SU2_MPI::Alltoallv(sb.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, in.data(), nR.data(), rD.data(), + MPI_UNSIGNED_LONG, comm); + }; + + vector inEdge, inWeight; + shipPairs(outEdge, inEdge); + shipPairs(outWeight, inWeight); + + /*--- Assemble the contracted graph for the columns owned here. ---*/ + const auto myFirstCol = static_cast(cvtxdist[rank]); + const auto nColMine = static_cast(cvtxdist[rank + 1] - cvtxdist[rank]); + + vector> cadj(nColMine); + for (size_t i = 0; i + 1 < inEdge.size(); i += 2) { + const auto a = inEdge[i] - myFirstCol; + if (a < nColMine) cadj[a].push_back(inEdge[i + 1]); + } + for (auto& v : cadj) { + sort(v.begin(), v.end()); + v.erase(unique(v.begin(), v.end()), v.end()); + } + + vector cvwgt(nColMine, 0); + for (auto c : inWeight) + if (c - myFirstCol < nColMine) cvwgt[c - myFirstCol]++; + for (auto& w : cvwgt) w = max(w, 1); + + vector cxadj(nColMine + 1, 0), cadjncy; + for (unsigned long c = 0; c < nColMine; ++c) cxadj[c + 1] = cxadj[c] + static_cast(cadj[c].size()); + cadjncy.reserve(cxadj[nColMine]); + for (auto& v : cadj) + for (auto b : v) cadjncy.push_back(static_cast(b)); + + /*--- A column with no neighbours at all would make ParMETIS fail; that only happens on a mesh of + * one point per rank, but check rather than risk it. ---*/ + unsigned long nEdgeLocal = cadjncy.size(), nEdgeGlobal = 0; + SU2_MPI::Allreduce(&nEdgeLocal, &nEdgeGlobal, 1, MPI_UNSIGNED_LONG, MPI_SUM, comm); + + if (nEdgeGlobal > 0) { + idx_t cwgtflag = 2, cedgecut = 0; + vector cpart(max(nColMine, 1)); + + if (rank == MASTER_NODE) { + cout << "Calling ParMETIS on " << nColGlobal << " contracted columns (" << Global_nPointDomain << " points, " + << nSweeps << " sweeps)..."; + } + auto cerr_ = ParMETIS_V3_PartKway(cvtxdist.data(), cxadj.data(), cadjncy.data(), cvwgt.data(), nullptr, + &cwgtflag, &numflag, &ncon, &nparts, tpwgts.data(), &ubvec, options, &cedgecut, + cpart.data(), &comm); + if (cerr_ != METIS_OK) SU2_MPI::Error("Column partitioning failed.", CURRENT_FUNCTION); + if (rank == MASTER_NODE) cout << " complete (" << cedgecut << " column cuts)." << endl; + + /*--- Give every point the colour of its column. ---*/ + map myColColour; + for (unsigned long c = 0; c < nColMine; ++c) myColColour[myFirstCol + c] = static_cast(cpart[c]); + + vector> want(size); + for (auto c : colOfPoint) { + if (c >= nColGlobal) continue; + int owner = 0; + while ((owner + 1 < size) && (static_cast(c) >= cvtxdist[owner + 1])) owner++; + want[owner].push_back(c); + } + vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); + for (int r = 0; r < size; ++r) { + sort(want[r].begin(), want[r].end()); + want[r].erase(unique(want[r].begin(), want[r].end()), want[r].end()); + nS[r] = static_cast(want[r].size()); + } + SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); + for (int r = 0; r < size; ++r) { + sD[r + 1] = sD[r] + nS[r]; + rD[r + 1] = rD[r] + nR[r]; + } + vector qs(sD[size]), qr(rD[size]); + for (int r = 0; r < size; ++r) copy(want[r].begin(), want[r].end(), qs.begin() + sD[r]); + SU2_MPI::Alltoallv(qs.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, qr.data(), nR.data(), rD.data(), + MPI_UNSIGNED_LONG, comm); + vector as(rD[size]), ar(sD[size]); + for (size_t i = 0; i < qr.size(); ++i) { + const auto it = myColColour.find(qr[i]); + as[i] = (it == myColColour.end()) ? 0 : it->second; + } + SU2_MPI::Alltoallv(as.data(), nR.data(), rD.data(), MPI_UNSIGNED_LONG, ar.data(), nS.data(), sD.data(), + MPI_UNSIGNED_LONG, comm); + map colour; + for (int r = 0; r < size; ++r) + for (int i = sD[r]; i < sD[r + 1]; ++i) colour[qs[i]] = ar[i]; + + for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { + const auto it = colour.find(colOfPoint[iPoint]); + part[iPoint] = (it == colour.end()) ? 0 : static_cast(it->second); + } + coloured = true; + } else if (rank == MASTER_NODE) { + cout << "Column partitioning found no graph to cut, falling back to point partitioning." << endl; + } + } /*--- Calling ParMETIS ---*/ - if (rank == MASTER_NODE) cout << "Calling ParMETIS..."; - auto err = ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), - adjwgt.empty() ? nullptr : adjwgt.data(), &wgtflag, &numflag, &ncon, &nparts, - tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); - if (err != METIS_OK) SU2_MPI::Error("Partitioning failed.", CURRENT_FUNCTION); - if (rank == MASTER_NODE) { - cout << " graph partitioning complete (" << edgecut << " edge cuts)." << endl; + if (!coloured) { + if (rank == MASTER_NODE) cout << "Calling ParMETIS..."; + auto err = ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), + adjwgt.empty() ? nullptr : adjwgt.data(), &wgtflag, &numflag, &ncon, &nparts, + tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); + if (err != METIS_OK) SU2_MPI::Error("Partitioning failed.", CURRENT_FUNCTION); + if (rank == MASTER_NODE) { + cout << " graph partitioning complete (" << edgecut << " edge cuts)." << endl; + } } /*--- Store the results of the partitioning (note that this is local diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index 84f30a2aa3d..f01ba6d6c88 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -1390,6 +1390,13 @@ void CSysMatrix::BuildLineletPreconditioner(const CGeometry* geometr } END_SU2_OMP_SAFE_GLOBAL_ACCESS + /*--- A rank whose part of the mesh holds no solid wall gets no linelets, and the working vectors + * above were deliberately not allocated for it. It still reaches this function, because the + * linear solver is collective, so it has to leave before indexing them. Partitioning a mesh + * over enough ranks eventually gives one of them no wall, which is why this only ever showed up + * beyond a couple of ranks. ---*/ + if (LineletUpper.empty()) return; + SU2_OMP_FOR_STAT(1) for (int iThread = 0; iThread < nThreads; ++iThread) { const auto size = CGeometry::CLineletInfo::MAX_LINELET_POINTS; diff --git a/config_template.cfg b/config_template.cfg index 2fdad98b989..94fd00aa75d 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1736,14 +1736,23 @@ MG_MIN_MESHSIZE= 500 % Enable agglomeration along implicit lines seeded from viscous walls (NO, YES) MG_IMPLICIT_LINES= NO % -% Maximum nodes on a wall-normal implicit agglomeration line, including the wall seed. -% Increase to extend the line deeper into the boundary layer (default 20). -MG_IMPLICIT_LINES_MAX_LENGTH= 20 -% -% Use isotropic (vs anisotropic) agglomeration for implicit lines (NO, YES) -% Anisotropic (NO): Pair cells normal to wall (2 cells per coarse CV, reduces mesh ~2x) -% Isotropic (YES): Pair cells in all directions (4 cells per coarse CV, reduces mesh ~4x) -MG_IMPLICIT_LINES_ISOTROPIC= NO +% Safety cap on how many layers deep a paving front may go. A front runs until it reaches a +% boundary or until the mesh stops offering a layer topologically identical to the one below it, +% so there is normally nothing to cap: 0 means no cap. +MG_IMPLICIT_LINES_MAX_LENGTH= 0 +% +% Grow boundary coarse CVs into the interior instead of leaving them one fine cell thick. +% The boundary agglomeration can only merge points that lie on the boundary themselves, so a +% boundary coarse CV comes out as a flat surface patch (2x2 nodes on a surface, 2 on a ridge) +% rather than the 2x2x2 block the interior pass builds. Those CVs are a large share of the +% coarse grid by count while holding very few nodes each. Thickening keeps the surface footprint +% and only adds the layer underneath it, so a CV still never straddles two boundary conditions. +% It is skipped where the mesh carries a stretched layer normal to the boundary, i.e. where the +% aspect ratio measured ALONG THE BOUNDARY NORMAL reaches this value: there the flat CV is +% deliberate semi-coarsening that preserves boundary-layer resolution, so viscous walls are left +% alone while symmetry planes and far fields in isotropic mesh are filled out. Raising the value +% thickens more boundaries, lowering it fewer. 0.0 disables thickening (historical behaviour). +MG_BOUNDARY_THICKEN_AR= 0.0 % % Number of iterations spent on each mesh during the Full Multigrid (FMG) startup phase. After % this many iterations the solution is prolongated to the next finer mesh. It is also the length From 513d8e8254cd70bc307470e9b2ea104ed43ce65a Mon Sep 17 00:00:00 2001 From: bigfooted Date: Wed, 2 Sep 2026 22:40:45 +0200 Subject: [PATCH 38/54] advancing front implicit line method --- .../include/geometry/CMultiGridGeometry.hpp | 18 +-- Common/src/geometry/CMultiGridGeometry.cpp | 121 ++---------------- 2 files changed, 11 insertions(+), 128 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index 601fb891d69..c80d0c3b4eb 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -48,20 +48,7 @@ class CMultiGridGeometry final : public CGeometry { * \return TRUE or FALSE depending if the control volume can be agglomerated. */ bool SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, const CGeometry* fine_grid, - const CConfig* config, const vector& mixedBC) const; - - /*! - * \brief Nodes carrying two or more physical boundary conditions of DIFFERENT type, e.g. the point - * where a wall ends against an outlet. Nishikawa's rules never agglomerate these: merging one - * into a coarse control volume averages two conditions that the fine grid applies separately, - * and neither ends up applied where it belongs. Two markers of the SAME type meeting - two - * wall patches, say - are not affected, nor is a node whose only second marker is - * SEND_RECEIVE, which records a partition and not a boundary condition. - * \param[in] fine_grid - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \return One flag per fine grid point, set where that point must stay on its own. - */ - vector FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const; + const CConfig* config) const; /*! * \brief Determine if a Point can be agglomerated using geometrical criteria. @@ -168,11 +155,10 @@ class CMultiGridGeometry final : public CGeometry { * \param[in] seeds - Seed nodes from SeedFrontNodes. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Definition of the particular problem. - * \param[in] mixedBC - Nodes that must stay on their own, from FindMixedBoundaryNodes. * \return One vector of indices into seeds.node per patch. */ vector> BuildFrontPatches(const CFrontSeeds& seeds, const CGeometry* fine_grid, - const CConfig* config, const vector& mixedBC) const; + const CConfig* config) const; public: /*--- This is to suppress Woverloaded-virtual, omitting it has no negative impact. ---*/ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index a9c29f23961..bdef3aa82ef 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -109,13 +109,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) onPhysBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } - /*--- Nodes where two different boundary conditions meet. Nishikawa's rules never agglomerate - * these, and the rule has to hold for every phase or the same node is treated one way by the - * paving and another here. In 2D the corner test below already refused them; in 3D a ridge of - * such nodes carries one identical marker PAIR all along it and would otherwise pair up with - * itself quite happily. ---*/ - const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); - vector bmembers; vector nThicken_DBG(fine_grid->GetnMarker(), 0), nFlat_DBG(fine_grid->GetnMarker(), 0); @@ -289,11 +282,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (counter > 2) agglomerate_seed = false; - /*--- ...and so is a node where two markers of DIFFERENT type meet, whatever the count and - * whatever the dimension. A coarse CV holding such a node would average two conditions the - * fine grid applies separately. ---*/ - if (mixedBC[iPoint]) agglomerate_seed = false; - /*--- If the seed (parent) can be agglomerated, we try to agglomerate connected childs to the parent ---*/ /*--- Note that in 2D we allow a maximum of 4 nodes to be agglomerated ---*/ @@ -303,7 +291,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : fine_grid->nodes->GetPoints(iPoint)) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -338,7 +326,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : Suitable_Indirect_Neighbors) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -775,21 +763,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, but only for the CVs that really do border another rank rather than for every CV near one. ---*/ - /*--- Coarse CVs that must be left exactly as the agglomeration made them: those holding a node - * where two different boundary conditions meet. Both repair passes below exist to get rid of - * one-child control volumes, and a deliberately isolated junction IS a one-child control volume, - * so without this they undo the isolation - the wall/symmetry node of a flat plate came out - * correctly alone and was then merged straight back into the CV above it. They have to be - * protected as a TARGET as well as a source: pass two merges a singleton into its smallest - * neighbour, and a one-child junction CV is by construction the smallest neighbour there is. ---*/ - vector mustStayAlone(nPointDomain, false); - for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) - for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) - if (mixedBC[nodes->GetChildren_CV(iCoarsePoint, iChildren)]) { - mustStayAlone[iCoarsePoint] = true; - break; - } - vector touchesPartition(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -805,12 +778,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { - if (mustStayAlone[iCoarsePoint]) continue; if ((nodes->GetnPoint(iCoarsePoint) == 1) && !touchesPartition[iCoarsePoint]) { /*--- Find the neighbor of the isolated point. This neighbor is the right control volume ---*/ const auto iCoarsePoint_Complete = nodes->GetPoint(iCoarsePoint, 0); - if (mustStayAlone[iCoarsePoint_Complete]) continue; /*--- Check if merging would exceed the maximum agglomeration size ---*/ auto nChildren_Target = nodes->GetnChildren_CV(iCoarsePoint_Complete); @@ -824,7 +795,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint_Complete)) { if (nChildrenToRedistribute == 0) break; - if (mustStayAlone[jCoarsePoint]) continue; auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); if (nChildren_Neighbor < maxAgglomSize) { @@ -903,7 +873,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; - if (mustStayAlone[iCoarsePoint]) continue; if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the @@ -915,7 +884,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long best_neighbor = std::numeric_limits::max(); unsigned short best_nChildren = 0; for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { - if (mustStayAlone[jCoarsePoint]) continue; const auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); /*--- Skip neighbors already emptied by an earlier merge in this same pass. ---*/ if (nChildren_Neighbor == 0) continue; @@ -1247,35 +1215,8 @@ bool CMultiGridGeometry::GeometricalCheck(unsigned long iPoint, const CGeometry* return (Volume); } -vector CMultiGridGeometry::FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const { - vector mixed(fine_grid->GetnPoint(), 0); - - /*--- The first physical condition seen at each node, -1 until one is. A second one of a different - * KIND is what makes the node mixed. A second marker of the same kind - two wall patches meeting - * - is not, and neither is SEND_RECEIVE, which records where the partition runs and says nothing - * about the boundary condition. ---*/ - vector firstBC(fine_grid->GetnPoint(), -1); - - for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - const auto bc = static_cast(config->GetMarker_All_KindBC(iMarker)); - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; - for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { - const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); - if (firstBC[iPoint] < 0) - firstBC[iPoint] = bc; - else if (firstBC[iPoint] != bc) - mixed[iPoint] = 1; - } - } - return mixed; -} - bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, - const CGeometry* fine_grid, const CConfig* config, - const vector& mixedBC) const { - /*--- A node where two boundary conditions of different type meet is never merged with anything. ---*/ - if (mixedBC[CVPoint]) return false; - + const CGeometry* fine_grid, const CConfig* config) const { bool agglomerate_CV = false; /*--- Basic condition, the point has not been previously agglomerated, it belongs to the domain, @@ -1993,8 +1934,8 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet } vector> CMultiGridGeometry::BuildFrontPatches(const CFrontSeeds& seeds, - const CGeometry* fine_grid, const CConfig* config, - const vector& mixedBC) const { + const CGeometry* fine_grid, + const CConfig* config) const { /*--- Every seed must end up in exactly one patch, and a patch must be compact: in 3D the four nodes * of one boundary quadrilateral, in 2D the two ends of one boundary edge. Choosing, for each seed * independently, a set of neighbours to merge with does not do this - the relation is not @@ -2061,26 +2002,6 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront vector groupOf(nSeeds); groups.reserve(nSeeds); - /*--- Seeds that sit next to an isolated junction node. A run of seeds between two junctions has - * whatever length the geometry gives it, and when that length is odd one seed cannot pair. Left - * to the plain index order the odd one out always lands at the END of the run, because that is - * where the sweep runs out - and the end of a run IS a junction. The result is a one-wide stack - * rising the full height of the mesh immediately beside the junction, so the coarse grid reads - * [4][4][2][1] on one side of it and [1][4][4] on the other. On a flat plate that puts the - * defect against the leading edge, which is the last place it should be. - * - * Pairing these first pushes the odd one out into the interior of the run, where a slightly - * narrower stack costs nothing, and leaves both sides of every junction matching. ---*/ - vector nextToMixed(nSeeds, 0); - for (unsigned long si = 0; si < nSeeds; ++si) { - if (mixedBC[seeds.node[si]]) continue; - for (auto jPoint : fine_grid->nodes->GetPoints(seeds.node[si])) - if (mixedBC[jPoint]) { - nextToMixed[si] = 1; - break; - } - } - /*--- Every seed starts as its own group; the rounds below merge them. ---*/ for (unsigned long si = 0; si < nSeeds; ++si) { groupOf[si] = si; @@ -2108,12 +2029,6 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront for (unsigned round = 0; round < nRounds; ++round) { const auto nGroups = groups.size(); - auto groupNextToMixed = [&](unsigned long g) { - for (auto si : groups[g]) - if (nextToMixed[si]) return true; - return false; - }; - /*--- Sort key of each group: the smallest global point index it holds. ---*/ gkey.assign(nGroups, std::numeric_limits::max()); for (unsigned long g = 0; g < nGroups; ++g) @@ -2123,17 +2038,11 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront * so the count from g's side equals the count from h's, and taking only h > g avoids both. ---*/ merges.clear(); for (unsigned long g = 0; g < nGroups; ++g) { - /*--- A node where two different boundary conditions meet stays a patch of its own, so the front - * rising from it is one node wide and its coarse CVs never reach across the junction. Both - * sides of the merge are tested: skipping only the mixed group would still let an ordinary - * group reach out and take it. ---*/ - if (mixedBC[seeds.node[groups[g].front()]]) continue; shared.clear(); for (auto si : groups[g]) for (auto sj : adj[si]) { const auto h = groupOf[sj]; if (h <= g) continue; - if (mixedBC[seeds.node[groups[h].front()]]) continue; if (groups[g].size() + groups[h].size() > max_group) continue; if (!sameSig(groups[h].front(), groups[g].front())) continue; @@ -2146,13 +2055,7 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront } if (!seen) shared.emplace_back(h, 1); } - /*--- Shared adjacency is doubled so a junction-adjacent merge can be ranked above another of - * the same shape without ever outranking a better shape: a square scores 4 or 5 and a strip - * 2 or 3, so squares still come first and the junction only breaks ties among equals. ---*/ - const unsigned long boostG = groupNextToMixed(g); - for (const auto& t : shared) - merges.push_back( - {g, t.first, 2 * t.second + (boostG || groupNextToMixed(t.first) ? 1u : 0u), gkey[g], gkey[t.first]}); + for (const auto& t : shared) merges.push_back({g, t.first, t.second, gkey[g], gkey[t.first]}); } /*--- Best first, over ALL groups at once. Sweeping the groups in index order instead and letting @@ -2269,8 +2172,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- PHASE 1. SeedFrontNodes must be reached by every rank, including one that owns no boundary: * it takes a collective to agree on which markers carry a layer. ---*/ const auto seeds = SeedFrontNodes(fine_grid, config, stiff); - const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); - const auto patches = BuildFrontPatches(seeds, fine_grid, config, mixedBC); + const auto patches = BuildFrontPatches(seeds, fine_grid, config); /*--- Nodes on a boundary that carries a boundary condition. A front must not grow into one: those * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two @@ -2473,12 +2375,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, pending[f] = front[f]; pendingLayers[f] = 1; - /*--- A front rooted on a node where two different boundary conditions meet emits that node as a - * coarse CV of its own, one fine node deep, so the junction is never averaged into a control - * volume with anything else. Only the FIRST CV of the stack is treated this way: emit() then - * asks blockFor again for what is by then an ordinary interior layer, and the rest of the - * stack rises two nodes at a time like any other. ---*/ - nBlock[f] = mixedBC[front[f].front()] ? 1 : blockFor(front[f]); + nBlock[f] = blockFor(front[f]); if (pendingLayers[f] >= nBlock[f]) emit(f); } @@ -2756,7 +2653,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, << " layers, front depth " << depthMin << " to " << depthMax; if (total[4] + total[5] > 0) cout << "\n Coarse CVs by depth: " << total[5] << " two layers deep, " << total[4] - << " one layer (isolated junction, or top of a stack)"; + << " one layer (top of a stack)"; /*--- Why each front stopped. Reaching a boundary is the one correct stop; everything else is the * mesh failing to offer a layer topologically identical to the current one, broken down by how * it failed. COLLISION and PINCH are two fronts, or two nodes of one front, reaching for the From a9cd513d49539f6f5d81a49b595fc99078fef6fc Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 4 Sep 2026 09:41:02 +0200 Subject: [PATCH 39/54] first fix mpi --- .../include/geometry/CMultiGridGeometry.hpp | 18 +- Common/src/geometry/CMultiGridGeometry.cpp | 537 ++++++++++++++---- 2 files changed, 454 insertions(+), 101 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index c80d0c3b4eb..601fb891d69 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -48,7 +48,20 @@ class CMultiGridGeometry final : public CGeometry { * \return TRUE or FALSE depending if the control volume can be agglomerated. */ bool SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, const CGeometry* fine_grid, - const CConfig* config) const; + const CConfig* config, const vector& mixedBC) const; + + /*! + * \brief Nodes carrying two or more physical boundary conditions of DIFFERENT type, e.g. the point + * where a wall ends against an outlet. Nishikawa's rules never agglomerate these: merging one + * into a coarse control volume averages two conditions that the fine grid applies separately, + * and neither ends up applied where it belongs. Two markers of the SAME type meeting - two + * wall patches, say - are not affected, nor is a node whose only second marker is + * SEND_RECEIVE, which records a partition and not a boundary condition. + * \param[in] fine_grid - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \return One flag per fine grid point, set where that point must stay on its own. + */ + vector FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const; /*! * \brief Determine if a Point can be agglomerated using geometrical criteria. @@ -155,10 +168,11 @@ class CMultiGridGeometry final : public CGeometry { * \param[in] seeds - Seed nodes from SeedFrontNodes. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Definition of the particular problem. + * \param[in] mixedBC - Nodes that must stay on their own, from FindMixedBoundaryNodes. * \return One vector of indices into seeds.node per patch. */ vector> BuildFrontPatches(const CFrontSeeds& seeds, const CGeometry* fine_grid, - const CConfig* config) const; + const CConfig* config, const vector& mixedBC) const; public: /*--- This is to suppress Woverloaded-virtual, omitting it has no negative impact. ---*/ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index bdef3aa82ef..d858bd2b8e7 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -109,6 +109,12 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) onPhysBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } + /*--- Nodes where two different boundary conditions meet. The rule has to hold for every phase or + * the same node is treated one way by the paving and another here. In 2D the corner test below + * already refused them; in 3D a ridge of such nodes carries one identical marker PAIR all along + * it and would otherwise pair up with itself quite happily. ---*/ + const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); + vector bmembers; vector nThicken_DBG(fine_grid->GetnMarker(), 0), nFlat_DBG(fine_grid->GetnMarker(), 0); @@ -282,6 +288,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (counter > 2) agglomerate_seed = false; + /*--- ...and so is a node where two markers of DIFFERENT type meet, whatever the count and + * whatever the dimension. A coarse CV holding such a node would average two conditions the + * fine grid applies separately. ---*/ + if (mixedBC[iPoint]) agglomerate_seed = false; + /*--- If the seed (parent) can be agglomerated, we try to agglomerate connected childs to the parent ---*/ /*--- Note that in 2D we allow a maximum of 4 nodes to be agglomerated ---*/ @@ -291,7 +302,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : fine_grid->nodes->GetPoints(iPoint)) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -326,7 +337,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto CVPoint : Suitable_Indirect_Neighbors) { /*--- The new point can be agglomerated ---*/ - if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config)) { + if (SetBoundAgglomeration(CVPoint, marker_seed, fine_grid, config, mixedBC)) { /*--- We set the value of the parent ---*/ fine_grid->nodes->SetParent_CV(CVPoint, Index_CoarseCV); @@ -693,57 +704,97 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } + /*--- Every rank owns a slice of the mesh, so every rank has its own share of these counts. Summing + * them and printing once is the only version that means anything: rank 0's slice is not the + * grid, and printing from all ranks interleaves the lines into each other - at eight ranks the + * output was literally unreadable, with one rank's histogram spliced through the middle of + * another's. Values are packed into one buffer so the whole report costs two collectives. ---*/ { - auto histOf = [&](unsigned long lo, unsigned long hi, const char* name) { - unsigned long h[10] = {0}, tot = 0, nod = 0; - unsigned long cube8 = 0, noncube8 = 0, square4 = 0, strip4 = 0; - unsigned long e8[14] = {0}; + constexpr unsigned NPHASE = 5, NPV = 30; /*--- Phases, and values accumulated per phase. ---*/ + /*--- Layout within a phase: [0..9] children histogram, [10] CVs, [11] nodes, [12] cubes, + * [13] slabs, [14] squares, [15] strips, [16..29] internal-edge histogram of the 8s. ---*/ + vector acc(NPHASE * NPV + 5, 0); + + auto histOf = [&](unsigned long lo, unsigned long hi, unsigned phase) { + auto* a = &acc[phase * NPV]; for (auto c = lo; c < hi; ++c) { const auto n = nodes->GetnChildren_CV(c); const auto e = intEdges_DBG[c]; if (n == 8) { - ((e >= 12) ? cube8 : noncube8)++; - e8[std::min(e, 13)]++; + a[(e >= 12) ? 12 : 13]++; + a[16 + std::min(e, 13)]++; } - if (n == 4) ((e >= 4) ? square4 : strip4)++; - h[std::min(n, 9)]++; - tot++; - nod += n; + if (n == 4) a[(e >= 4) ? 14 : 15]++; + a[std::min(n, 9)]++; + a[10]++; + a[11] += n; } - if (tot == 0) return; - cout << " " << name << ": " << tot << " CVs, " << nod << " nodes, avg " << (su2double(nod) / su2double(tot)) - << " sizes"; - for (unsigned s = 1; s <= 9; ++s) - if (h[s] > 0) cout << " " << s << ":" << h[s]; - if (h[8] > 0) { - cout << " [of the 8s: " << cube8 << " are 2x2x2 cubes, " << noncube8 << " are slabs/strips; internal edges"; - for (unsigned e = 7; e <= 13; ++e) - if (e8[e] > 0) cout << " " << (e == 13 ? ">=13" : to_string(e)) << ":" << e8[e]; - cout << "]"; - } - if (h[4] > 0) cout << " [of the 4s: " << square4 << " square, " << strip4 << " strip]"; - cout << endl; }; - unsigned long degSum = 0, degN = 0; + histOf(0, starting_idx_lines_DBG, 0); + histOf(starting_idx_lines_DBG, idx_after_lines_DBG, 1); + histOf(idx_after_lines_DBG, idx_after_bound_DBG, 2); + histOf(idx_after_bound_DBG, idx_after_domain_DBG, 3); + histOf(idx_after_domain_DBG, Index_CoarseCV, 4); + for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { if (!fine_grid->nodes->GetDomain(iPoint)) continue; - degSum += fine_grid->nodes->GetnPoint(iPoint); - degN++; - } - cout << " CV size distribution by phase (maxAgglomSize=" << maxAgglomSize - << ", mean fine-graph degree=" << (su2double(degSum) / su2double(max(degN, 1ul))) - << ", a 2x2x2 block has 12 internal edges):" << endl; - histOf(0, starting_idx_lines_DBG, "pre-lines "); - histOf(starting_idx_lines_DBG, idx_after_lines_DBG, "implicit lines "); - histOf(idx_after_lines_DBG, idx_after_bound_DBG, "boundary STEP1 "); - histOf(idx_after_bound_DBG, idx_after_domain_DBG, "domain STEP2 "); - histOf(idx_after_domain_DBG, Index_CoarseCV, "leftover single"); - cout << " STEP2 rejected seeds: " << nRejectedSeed_DBG << ", iterations used: " << iteration << "/" - << fine_grid->GetnPoint() << endl; + acc[NPHASE * NPV + 0] += fine_grid->nodes->GetnPoint(iPoint); + acc[NPHASE * NPV + 1]++; + } + acc[NPHASE * NPV + 2] = nRejectedSeed_DBG; + acc[NPHASE * NPV + 3] = iteration; + acc[NPHASE * NPV + 4] = fine_grid->GetnPointDomain(); + + vector tot(acc.size(), 0); + SU2_MPI::Allreduce(acc.data(), tot.data(), static_cast(acc.size()), MPI_UNSIGNED_LONG, MPI_SUM, + SU2_MPI::GetComm()); + + /*--- Thickening is counted per configuration-file marker, not per local marker: ranks agree on + * neither the number nor the order of local markers, because each appends its own + * SEND_RECEIVE markers, so the same index means a different boundary elsewhere. ---*/ + const auto nMarkerCfg = config->GetnMarker_CfgFile(); + vector mk(2 * nMarkerCfg, 0), mkTot(2 * nMarkerCfg, 0); for (auto m = 0u; m < fine_grid->GetnMarker(); ++m) { - if (nThicken_DBG[m] + nFlat_DBG[m] == 0) continue; - cout << " marker " << config->GetMarker_All_TagBound(m) << ": thickened " << nThicken_DBG[m] << ", kept flat " - << nFlat_DBG[m] << endl; + if (config->GetMarker_All_KindBC(m) == SEND_RECEIVE) continue; + const auto c = config->GetMarker_CfgFile_TagBound(config->GetMarker_All_TagBound(m)); + mk[2 * c] += nThicken_DBG[m]; + mk[2 * c + 1] += nFlat_DBG[m]; + } + if (nMarkerCfg > 0) + SU2_MPI::Allreduce(mk.data(), mkTot.data(), static_cast(mk.size()), MPI_UNSIGNED_LONG, MPI_SUM, + SU2_MPI::GetComm()); + + if (rank == MASTER_NODE) { + const char* phaseName[NPHASE] = {"pre-lines ", "implicit lines ", "boundary STEP1 ", "domain STEP2 ", + "leftover single"}; + const auto degN = std::max(tot[NPHASE * NPV + 1], 1ul); + cout << " CV size distribution by phase (maxAgglomSize=" << maxAgglomSize + << ", mean fine-graph degree=" << (su2double(tot[NPHASE * NPV + 0]) / su2double(degN)) + << ", a 2x2x2 block has 12 internal edges):" << endl; + + for (unsigned ph = 0; ph < NPHASE; ++ph) { + const auto* a = &tot[ph * NPV]; + if (a[10] == 0) continue; + cout << " " << phaseName[ph] << ": " << a[10] << " CVs, " << a[11] << " nodes, avg " + << (su2double(a[11]) / su2double(a[10])) << " sizes"; + for (unsigned sz = 1; sz <= 9; ++sz) + if (a[sz] > 0) cout << " " << sz << ":" << a[sz]; + if (a[8] > 0) { + cout << " [of the 8s: " << a[12] << " are 2x2x2 cubes, " << a[13] << " are slabs/strips; internal edges"; + for (unsigned e = 7; e <= 13; ++e) + if (a[16 + e] > 0) cout << " " << (e == 13 ? ">=13" : to_string(e)) << ":" << a[16 + e]; + cout << "]"; + } + if (a[4] > 0) cout << " [of the 4s: " << a[14] << " square, " << a[15] << " strip]"; + cout << endl; + } + cout << " STEP2 rejected seeds: " << tot[NPHASE * NPV + 2] << ", iterations used: " << tot[NPHASE * NPV + 3] + << "/" << tot[NPHASE * NPV + 4] << endl; + for (auto c = 0u; c < nMarkerCfg; ++c) { + if (mkTot[2 * c] + mkTot[2 * c + 1] == 0) continue; + cout << " marker " << config->GetMarker_CfgFile_TagBound(c) << ": thickened " << mkTot[2 * c] + << ", kept flat " << mkTot[2 * c + 1] << endl; + } } } @@ -763,6 +814,27 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, but only for the CVs that really do border another rank rather than for every CV near one. ---*/ + /*--- Coarse CVs that must be left exactly as the agglomeration made them: those holding a node + * where two different boundary conditions meet. Both repair passes below exist to get rid of + * one-child control volumes, and a deliberately isolated junction IS a one-child control volume, + * so without this they simply undo the isolation. They have to be protected as a TARGET as well + * as a source: pass two merges a singleton into its smallest neighbour, and a one-child junction + * CV is by construction the smallest neighbour there is. ---*/ + vector mustStayAlone(nPointDomain, false); + /*--- ...and which coarse CVs hold a boundary node at all. A boundary node is never agglomerated + * with an interior one, which the agglomeration itself already respects - the first advance of + * every front is a single layer, so the boundary layer is its own CV. The repair passes below + * would otherwise undo it from the other end: they merge a one-child CV into a neighbour, and a + * boundary CV's neighbours include the interior CV sitting on top of it. On the flat plate they + * happened to pick a boundary neighbour every time, which is luck rather than a rule. ---*/ + vector cvOnBoundary(nPointDomain, false); + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) + for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { + const auto iFinePoint = nodes->GetChildren_CV(iCoarsePoint, iChildren); + if (mixedBC[iFinePoint]) mustStayAlone[iCoarsePoint] = true; + if (onPhysBoundary[iFinePoint]) cvOnBoundary[iCoarsePoint] = true; + } + vector touchesPartition(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -778,10 +850,13 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { + if (mustStayAlone[iCoarsePoint]) continue; if ((nodes->GetnPoint(iCoarsePoint) == 1) && !touchesPartition[iCoarsePoint]) { /*--- Find the neighbor of the isolated point. This neighbor is the right control volume ---*/ const auto iCoarsePoint_Complete = nodes->GetPoint(iCoarsePoint, 0); + if (mustStayAlone[iCoarsePoint_Complete]) continue; + if (cvOnBoundary[iCoarsePoint] != cvOnBoundary[iCoarsePoint_Complete]) continue; /*--- Check if merging would exceed the maximum agglomeration size ---*/ auto nChildren_Target = nodes->GetnChildren_CV(iCoarsePoint_Complete); @@ -795,6 +870,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint_Complete)) { if (nChildrenToRedistribute == 0) break; + if (mustStayAlone[jCoarsePoint]) continue; + if (cvOnBoundary[jCoarsePoint] != cvOnBoundary[iCoarsePoint_Complete]) continue; auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); if (nChildren_Neighbor < maxAgglomSize) { @@ -873,6 +950,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; + if (mustStayAlone[iCoarsePoint]) continue; if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the @@ -884,6 +962,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned long best_neighbor = std::numeric_limits::max(); unsigned short best_nChildren = 0; for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { + if (mustStayAlone[jCoarsePoint]) continue; + if (cvOnBoundary[jCoarsePoint] != cvOnBoundary[iCoarsePoint]) continue; const auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); /*--- Skip neighbors already emptied by an earlier merge in this same pass. ---*/ if (nChildren_Neighbor == 0) continue; @@ -1186,7 +1266,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * because those still move fine points between coarse CVs - a dump taken before them shows what * the agglomeration intended rather than what the solver will actually use. ---*/ if (getenv("DUMPAGGLOM") != nullptr) { - ofstream fdump(string("agglom_level") + to_string(iMesh) + ".dat"); + ofstream fdump(string("agglom_level") + to_string(iMesh) + "_r" + to_string(rank) + ".dat"); for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { if (!fine_grid->nodes->GetDomain(iPoint)) continue; const auto* c = fine_grid->nodes->GetCoord(iPoint); @@ -1215,8 +1295,35 @@ bool CMultiGridGeometry::GeometricalCheck(unsigned long iPoint, const CGeometry* return (Volume); } +vector CMultiGridGeometry::FindMixedBoundaryNodes(const CGeometry* fine_grid, const CConfig* config) const { + vector mixed(fine_grid->GetnPoint(), 0); + + /*--- The first physical condition seen at each node, -1 until one is. A second one of a different + * KIND is what makes the node mixed. A second marker of the same kind - two wall patches meeting + * - is not, and neither is SEND_RECEIVE, which records where the partition runs and says nothing + * about the boundary condition. ---*/ + vector firstBC(fine_grid->GetnPoint(), -1); + + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + const auto bc = static_cast(config->GetMarker_All_KindBC(iMarker)); + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { + const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (firstBC[iPoint] < 0) + firstBC[iPoint] = bc; + else if (firstBC[iPoint] != bc) + mixed[iPoint] = 1; + } + } + return mixed; +} + bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vector marker_seed, - const CGeometry* fine_grid, const CConfig* config) const { + const CGeometry* fine_grid, const CConfig* config, + const vector& mixedBC) const { + /*--- A node where two boundary conditions of different type meet is never merged with anything. ---*/ + if (mixedBC[CVPoint]) return false; + bool agglomerate_CV = false; /*--- Basic condition, the point has not been previously agglomerated, it belongs to the domain, @@ -1921,6 +2028,12 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet nQualified.swap(tmp); } + if ((rank == MASTER_NODE) && (getenv("DUMPAGGLOM") != nullptr)) + for (auto iCfg = 0u; iCfg < nMarkerCfg; ++iCfg) + if (nValid[iCfg] > 0) + cout << " Seed vote " << config->GetMarker_CfgFile_TagBound(iCfg) << ": " << nQualified[iCfg] << "/" + << nValid[iCfg] << " = " << (su2double(nQualified[iCfg]) / su2double(nValid[iCfg])) << endl; + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { if (!canSeed(iMarker)) continue; const auto iCfg = cfgOfMarker[iMarker]; @@ -1934,8 +2047,8 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet } vector> CMultiGridGeometry::BuildFrontPatches(const CFrontSeeds& seeds, - const CGeometry* fine_grid, - const CConfig* config) const { + const CGeometry* fine_grid, const CConfig* config, + const vector& mixedBC) const { /*--- Every seed must end up in exactly one patch, and a patch must be compact: in 3D the four nodes * of one boundary quadrilateral, in 2D the two ends of one boundary edge. Choosing, for each seed * independently, a set of neighbours to merge with does not do this - the relation is not @@ -2038,11 +2151,16 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront * so the count from g's side equals the count from h's, and taking only h > g avoids both. ---*/ merges.clear(); for (unsigned long g = 0; g < nGroups; ++g) { + /*--- A node where two different boundary conditions meet stays a patch of its own, so the front + * rising from it is one node wide. Both sides of a merge are tested: skipping only the mixed + * group would still let an ordinary group reach out and take it. ---*/ + if (mixedBC[seeds.node[groups[g].front()]]) continue; shared.clear(); for (auto si : groups[g]) for (auto sj : adj[si]) { const auto h = groupOf[sj]; if (h <= g) continue; + if (mixedBC[seeds.node[groups[h].front()]]) continue; if (groups[g].size() + groups[h].size() > max_group) continue; if (!sameSig(groups[h].front(), groups[g].front())) continue; @@ -2172,7 +2290,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- PHASE 1. SeedFrontNodes must be reached by every rank, including one that owns no boundary: * it takes a collective to agree on which markers carry a layer. ---*/ const auto seeds = SeedFrontNodes(fine_grid, config, stiff); - const auto patches = BuildFrontPatches(seeds, fine_grid, config); + const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); + const auto patches = BuildFrontPatches(seeds, fine_grid, config, mixedBC); /*--- Nodes on a boundary that carries a boundary condition. A front must not grow into one: those * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two @@ -2218,21 +2337,41 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, su2double dir[MAXNDIM]; /*!< Unit step direction, reused to update the front's direction. */ }; - const auto nFront = patches.size(); - - vector> front(nFront), pending(nFront); - vector> dirNow(nFront); - vector alive(nFront, 0); - vector depth(nFront, 0), nBlock(nFront, 1), pendingLayers(nFront, 0); - vector> prop(nFront); + /*--- Fronts are not a fixed set: one handed over from a neighbouring rank is appended while the + * rounds are running, so every per-front array grows and the loops below are bounded by + * front.size() rather than by the number of patches. ---*/ + vector> front, pending, handTo; + vector> dirNow; + vector alive; + vector depth, nBlock, pendingLayers, tag; + vector> prop; + + /*--- A name for a front that means the same thing on every rank, so a stack handed across a + * partition can be recognised on the far side and so two ranks reaching for the same node can be + * separated the same way by both. The smallest global point index of the patch it grew from is + * unique, since a seed belongs to exactly one patch; the +1 leaves 0 free to mean "nothing". ---*/ + auto addFront = [&](const vector& layer, const std::array& dir, + unsigned long frontTag, unsigned long block) { + front.push_back(layer); + pending.push_back(layer); + handTo.emplace_back(); + dirNow.push_back(dir); + alive.push_back(1); + depth.push_back(0); + nBlock.push_back(block); + pendingLayers.push_back(1); + tag.push_back(frontTag); + prop.emplace_back(); + return front.size() - 1; + }; vector claimed(nPointFine, 0); /*--- Confirmed owner of a claimed node, -1 while free. Only ever written when a layer is accepted, * so a bid that is still being contested never appears here. ---*/ vector frontOf(nPointFine, -1); - vector failed(nFront, 0); - vector failReason(nFront, 0); + vector failed; + vector failReason; vector stopCounts(N_STOP_REASONS, 0); /*--- The bid table. Only the index is kept per mesh point, and the bids themselves live in a @@ -2249,6 +2388,31 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, vector newLayer; unsigned long nStacks = 0, nSemiCV = 0, nFullCV = 0, nLayers = 0, nCovered = 0; + unsigned long nHandedOut = 0, nHandedIn = 0; + unsigned long nStraddle = 0, nStraddleOwned = 0, nStraddleHalo = 0, nNoHalo = 0; + + /*--- One footprint node arriving from a neighbouring rank, to be regrouped by tag. ---*/ + struct CInherited { + unsigned long tag; /*!< \brief The front it belongs to, the same name on both ranks. */ + unsigned long node; /*!< \brief Local index of the node, owned by this rank. */ + su2double dir[MAXNDIM]; /*!< \brief The marching direction the stack arrives with. */ + }; + vector inherited; + + /*--- Where each halo node sits in this rank's SEND_RECEIVE receive lists, so a handover can be + * packed against the right vertex without searching. A halo node appears in exactly one. ---*/ + vector haloMarker(nPointFine, -1); + vector haloVertex(nPointFine, 0); + for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { + if (!((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0))) + continue; + const auto MarkerR = iMarker + 1; + for (auto iVertex = 0ul; iVertex < fine_grid->nVertex[MarkerR]; iVertex++) { + const auto p = fine_grid->vertex[MarkerR][iVertex]->GetNode(); + haloMarker[p] = static_cast(MarkerR); + haloVertex[p] = iVertex; + } + } unsigned long histogram[9] = {0}; auto markFail = [&](unsigned long f, unsigned short why) { @@ -2331,7 +2495,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- The boundary layer of every front. Claiming it before ordinary boundary agglomeration runs is * what keeps the stack aligned: every layer above has exactly the same footprint. ---*/ - for (unsigned long f = 0; f < nFront; ++f) { + for (const auto& patch : patches) { /*--- A one-node patch is allowed to seed a front and marches as a stack one node wide. It looks * like a poor footprint, but the alternative - leaving it to ordinary agglomeration - is far * worse: a seed that cannot pair is one whose whole COLUMN then goes unpaved, from the wall to @@ -2339,52 +2503,56 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * cannot pair for a reason. On the flat plate they were the two nodes at the leading edge, * where the wall meets the symmetry plane and the marker signatures differ, plus the two * domain corners: five full-height stripes cut through the paved region, the worst of them - * right at the leading edge. One-wide stacks leave no such hole, and since a coarse CV is now - * always two layers deep they still hold two nodes each rather than one. ---*/ - bool valid = !patches[f].empty(); - for (auto si : patches[f]) { + * right at the leading edge. ---*/ + bool valid = !patch.empty(); + for (auto si : patch) { const auto p = seeds.node[si]; if (!GeometricalCheck(p, fine_grid, config) || fine_grid->nodes->GetAgglomerate(p)) valid = false; } if (!valid) continue; + vector layer0; std::array n0{}; - for (auto si : patches[f]) { - front[f].push_back(seeds.node[si]); - claimed[seeds.node[si]] = 1; - frontOf[seeds.node[si]] = static_cast(f); + unsigned long frontTag = std::numeric_limits::max(); + for (auto si : patch) { + layer0.push_back(seeds.node[si]); + frontTag = std::min(frontTag, fine_grid->nodes->GetGlobalIndex(seeds.node[si])); for (unsigned short d = 0; d < nDim; ++d) n0[d] += seeds.normal[si][d]; } const su2double nrm = GeometryToolbox::Norm(nDim, n0.data()); - if (nrm <= 0.0) { - /*--- A patch whose members' normals cancel has no direction to march in. ---*/ - for (auto p : front[f]) { - claimed[p] = 0; - frontOf[p] = -1; - } - front[f].clear(); - continue; - } + /*--- A patch whose members' normals cancel has no direction to march in. ---*/ + if (nrm <= 0.0) continue; for (unsigned short d = 0; d < nDim; ++d) n0[d] /= nrm; - dirNow[f] = n0; - histogram[std::min(front[f].size(), 8)]++; - alive[f] = 1; + /*--- The boundary layer becomes a coarse CV on its own: a boundary node is never merged with an + * interior one, so the first advance of every front is a single layer. Only the first - emit() + * then asks blockFor again for what is by then an ordinary interior layer, and the rest of the + * stack rises two nodes at a time. This is also what isolates a junction node without needing + * a rule of its own: such a node is a patch of one, so its first CV holds it and nothing else. ---*/ + const auto f = addFront(layer0, n0, frontTag + 1, 1); + for (auto p : layer0) { + claimed[p] = 1; + frontOf[p] = static_cast(f); + } + histogram[std::min(layer0.size(), 8)]++; nStacks++; nLayers++; - - pending[f] = front[f]; - pendingLayers[f] = 1; - nBlock[f] = blockFor(front[f]); - if (pendingLayers[f] >= nBlock[f]) emit(f); + emit(f); } for (unsigned long layer = 1;; ++layer) { - bool anyAlive = false; - for (unsigned long f = 0; f < nFront; ++f) anyAlive = anyAlive || alive[f]; - if (!anyAlive) break; - - std::fill(failed.begin(), failed.end(), 0); + /*--- Whether ANY rank still has a live front, not just this one. Every rank has to run the same + * number of rounds because each round ends in a handover exchange that they all take part in: + * a rank whose own fronts are long finished may still be about to receive a stack from a + * neighbour, and a rank that dropped out of the loop early would hang the ones that did not. ---*/ + int aliveLocal = 0; + for (unsigned long f = 0; f < front.size(); ++f) aliveLocal |= alive[f]; + int aliveGlobal = 0; + SU2_MPI::Allreduce(&aliveLocal, &aliveGlobal, 1, MPI_INT, MPI_MAX, SU2_MPI::GetComm()); + if (aliveGlobal == 0) break; + + failed.assign(front.size(), 0); + failReason.assign(front.size(), 0); for (const auto& b : bids) bidIdx[b.node] = NOBID; bids.clear(); bidOwner.clear(); @@ -2392,9 +2560,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- (a) Every alive front proposes a successor for each of its nodes. A front that cannot fill a * whole layer proposes NOTHING: it is retiring this round anyway, and letting its partial bids * stand would let a dying front displace a healthy one out of nodes it can still use. ---*/ - for (unsigned long f = 0; f < nFront; ++f) { + for (unsigned long f = 0; f < front.size(); ++f) { if (!alive[f]) continue; prop[f].clear(); + handTo[f].clear(); if ((MAX_LINE_LENGTH > 0) && (depth[f] + 1 >= MAX_LINE_LENGTH)) { markFail(f, STOP_MAX_LENGTH); @@ -2404,6 +2573,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, for (auto n : front[f]) { auto best = NO_POINT; su2double best_dot = -2.0, best_len = 0.0, best_dir[MAXNDIM] = {0.0}; + /*--- The best step onto a node this rank does NOT own, kept separately. It cannot be claimed + * here, but it is where the stack would go next, so it is what gets handed over. ---*/ + auto bestHalo = NO_POINT; + su2double bestHalo_dot = -2.0; bool sawCollision = false, sawBoundary = false, sawAgglom = false, sawPartition = false; bool sawGeom = false; @@ -2428,6 +2601,13 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * stop that was really the partitioning. ---*/ if (!fine_grid->nodes->GetDomain(jPoint)) { sawPartition = true; + /*--- Held as a handover candidate, subject to the same admissibility the owner would + * apply anyway; whether it is still free is the owner's to decide. ---*/ + if (dot > bestHalo_dot && !(onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) && + GeometricalCheck(jPoint, fine_grid, config)) { + bestHalo_dot = dot; + bestHalo = jPoint; + } continue; } if (fine_grid->nodes->GetAgglomerate(jPoint)) { @@ -2461,15 +2641,24 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } + if ((best == NO_POINT) && (bestHalo != NO_POINT)) { + /*--- Nowhere left on this rank, but the stack does continue - just on someone else's side + * of the interface. Remember where, and let the classification below decide whether the + * whole layer goes over. ---*/ + handTo[f].push_back(bestHalo); + continue; + } + if (best == NO_POINT) { /*--- Priority order picks the cleanest explanation first: reaching a physical boundary is a * correct, expected stop and takes priority even if some other, non-viable candidate also * happened to be claimed. Only report a collision when no boundary was involved. ---*/ if (sawBoundary) markFail(f, STOP_PHYS_BOUNDARY); - else if (sawPartition) + else if (sawPartition) { + nNoHalo++; markFail(f, STOP_PARTITION); - else if (sawCollision) + } else if (sawCollision) markFail(f, STOP_COLLISION); else if (sawAgglom) markFail(f, STOP_AGGLOMERATED); @@ -2490,7 +2679,23 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, prop[f].push_back(s); } - if (failed[f]) prop[f].clear(); + /*--- A layer goes wholly to this rank or wholly to the neighbour. A front that would have to + * split across the interface retires: half a layer is not an extrusion, and stitching the + * two halves back together afterwards would need the two ranks to agree on a footprint + * neither of them holds in full. ---*/ + if (failed[f]) { + prop[f].clear(); + handTo[f].clear(); + } else if (!handTo[f].empty()) { + prop[f].clear(); + if (handTo[f].size() != front[f].size()) { + nStraddleOwned += front[f].size() - handTo[f].size(); + nStraddleHalo += handTo[f].size(); + nStraddle++; + handTo[f].clear(); + markFail(f, STOP_PARTITION); + } + } } /*--- (b) Contention resolved from bids that were all collected before any was granted, so the @@ -2502,7 +2707,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, return a.key < b.key; }; - for (unsigned long f = 0; f < nFront; ++f) { + for (unsigned long f = 0; f < front.size(); ++f) { if (!alive[f] || failed[f]) continue; for (const auto& s : prop[f]) { /*--- A front that has just lost a bid is retiring, so it must not go on to place the rest and @@ -2549,7 +2754,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- (c) All-or-nothing acceptance: a front takes the whole layer or none of it and retires. A * bid only becomes a claim here, so a retiring front never has to give anything back and the * nodes it was reaching for stay available to ordinary agglomeration. ---*/ - for (unsigned long f = 0; f < nFront; ++f) { + for (unsigned long f = 0; f < front.size(); ++f) { if (!alive[f]) continue; newLayer.clear(); @@ -2605,31 +2810,159 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, pendingLayers[f]++; if (pendingLayers[f] >= nBlock[f]) emit(f); } + + /*============================================================================================== + * (d) Hand stacks across partition interfaces. + * + * A front that has run into the halo cannot go on here: those nodes belong to another rank, and + * their parent is that rank's to assign. Without this the stack simply ended at the interface, + * and on four ranks that was 70 of 71 fronts - the paved fraction of the flat plate fell from + * 99% to 48% purely because of where the partition happened to cut. + * + * Instead the footprint is sent to the owner, which picks the stack up and carries on. What + * crosses is not a coarse CV - a CV belongs wholly to one rank - but the FOOTPRINT, so the two + * halves of the stack stay the same shape and the coarse grid reads the same across the seam. + * The handover travels the reverse of the usual halo direction: a node this rank sees as halo is + * one the neighbour owns, so it is packed against the RECEIVE marker and sent to the rank this + * marker normally receives from. + *============================================================================================*/ + for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { + if (!((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0))) + continue; + + const auto MarkerS = iMarker, MarkerR = iMarker + 1; + const auto send_to = config->GetMarker_All_SendRecv(MarkerS) - 1; + const auto receive_from = abs(config->GetMarker_All_SendRecv(MarkerR)) - 1; + const auto nVertexS = fine_grid->nVertex[MarkerS]; + const auto nVertexR = fine_grid->nVertex[MarkerR]; + + /*--- Packed against the halo vertices, i.e. what this rank wants the neighbour to continue. ---*/ + vector tagOut(nVertexR, 0), tagIn(nVertexS, 0); + vector dirOut(nVertexR * nDim, 0.0), dirIn(nVertexS * nDim, 0.0); + + for (unsigned long f = 0; f < front.size(); ++f) { + if (handTo[f].empty()) continue; + for (auto p : handTo[f]) { + if (haloMarker[p] != static_cast(MarkerR)) continue; + const auto v = haloVertex[p]; + /*--- Two fronts of this rank reaching for the same node: the lower tag takes it, which is + * a decision both ranks would reach the same way. ---*/ + if ((tagOut[v] != 0) && (tagOut[v] <= tag[f])) continue; + tagOut[v] = tag[f]; + for (unsigned short d = 0; d < nDim; ++d) dirOut[v * nDim + d] = dirNow[f][d]; + } + } + + SU2_MPI::Sendrecv(tagOut.data(), nVertexR, MPI_UNSIGNED_LONG, receive_from, 2, tagIn.data(), nVertexS, + MPI_UNSIGNED_LONG, send_to, 2, SU2_MPI::GetComm(), MPI_STATUS_IGNORE); + SU2_MPI::Sendrecv(dirOut.data(), nVertexR * nDim, MPI_DOUBLE, receive_from, 3, dirIn.data(), nVertexS * nDim, + MPI_DOUBLE, send_to, 3, SU2_MPI::GetComm(), MPI_STATUS_IGNORE); + + for (auto iVertex = 0ul; iVertex < nVertexS; iVertex++) { + if (tagIn[iVertex] == 0) continue; + CInherited h; + h.tag = tagIn[iVertex]; + h.node = fine_grid->vertex[MarkerS][iVertex]->GetNode(); + for (unsigned short d = 0; d < nDim; ++d) h.dir[d] = dirIn[iVertex * nDim + d]; + inherited.push_back(h); + } + } + + /*--- Every front that handed over is finished here; the neighbour owns the rest of the stack. ---*/ + for (unsigned long f = 0; f < front.size(); ++f) { + if (handTo[f].empty()) continue; + handTo[f].clear(); + alive[f] = 0; + stopCounts[STOP_PARTITION]++; + nHandedOut++; + emit(f); + } + + /*--- Adopt what the neighbours sent. Tags are processed in ascending order so that two ranks + * handing stacks onto overlapping nodes are separated the same way whatever order the messages + * happened to arrive in. A footprint whose nodes are not all still free is dropped: the stack + * simply ends, exactly as it would have before. ---*/ + std::sort(inherited.begin(), inherited.end(), + [](const CInherited& a, const CInherited& b) { return a.tag < b.tag; }); + + for (size_t i = 0; i < inherited.size();) { + size_t j = i; + while ((j < inherited.size()) && (inherited[j].tag == inherited[i].tag)) ++j; + + vector layer0; + bool ok = true; + for (size_t k = i; k < j; ++k) { + const auto p = inherited[k].node; + if (claimed[p] || fine_grid->nodes->GetAgglomerate(p) || !GeometricalCheck(p, fine_grid, config)) ok = false; + layer0.push_back(p); + } + /*--- The footprint has to arrive whole and connected, the same test any other layer passes. ---*/ + if (ok && (layer0.size() > 1)) { + vector seen(layer0.size(), 0); + vector stk{0}; + seen[0] = 1; + size_t nSeen = 1; + while (!stk.empty()) { + const auto cur = stk.back(); + stk.pop_back(); + for (size_t k = 0; k < layer0.size(); ++k) { + if (seen[k] || !isAdjacent(layer0[cur], layer0[k])) continue; + seen[k] = 1; + nSeen++; + stk.push_back(k); + } + } + if (nSeen != layer0.size()) ok = false; + } + + if (ok) { + std::array d0{}; + for (unsigned short d = 0; d < nDim; ++d) d0[d] = inherited[i].dir[d]; + /*--- An inherited layer is an interior one, so it is NOT subject to the single-layer rule the + * boundary layer gets: it opens an ordinary two-deep coarse CV and waits for its partner. ---*/ + const auto nf = addFront(layer0, d0, inherited[i].tag, blockFor(layer0)); + for (auto p : layer0) { + claimed[p] = 1; + frontOf[p] = static_cast(nf); + } + nLayers++; + nHandedIn++; + if (pendingLayers[nf] >= nBlock[nf]) emit(nf); + } + i = j; + } + inherited.clear(); } /*--- Nothing should be left buffered, but a front retired outside the loop would strand its nodes * with a parent index that was never assigned. ---*/ - for (unsigned long f = 0; f < nFront; ++f) emit(f); + for (unsigned long f = 0; f < front.size(); ++f) emit(f); /*--- How far each front actually got. This is the number to watch when the paved region does not * look like a front: fronts that all reach the same height leave a flat interface with ordinary * agglomeration, and a spread here is that interface coming out as a staircase instead. ---*/ unsigned long dmin = std::numeric_limits::max(), dmax = 0; - for (unsigned long f = 0; f < nFront; ++f) { + for (unsigned long f = 0; f < front.size(); ++f) { if (front[f].empty()) continue; dmin = std::min(dmin, depth[f]); dmax = std::max(dmax, depth[f]); } - if (dmin == std::numeric_limits::max()) dmin = 0; + /*--- A rank with no fronts of its own must not drag the reported minimum to zero: leaving dmin at + * its sentinel keeps it out of the MPI_MIN below, so the range describes the fronts that exist + * rather than the ranks that have none. ---*/ + if (getenv("DUMPAGGLOM") != nullptr) + cout << " Paving rank " << rank << ": " << front.size() << " fronts, depth " << dmin << ".." << dmax + << ", local nPointDomain " << fine_grid->GetnPointDomain() << endl; /*--- Summary over all ranks. Reporting rank 0's own fronts makes a partitioned run look like a * fraction of the mesh it is not, and hides how much of the layer the partitioning cost: a front * stops at the partition, so the number of nodes left to ordinary agglomeration is the number to * watch when adding ranks. Every rank must reach these collectives. ---*/ unsigned long nSeedNodes = seeds.node.size(); - unsigned long local[6] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV}; - unsigned long total[6] = {0}; - SU2_MPI::Allreduce(local, total, 6, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + unsigned long local[12] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV, + nHandedOut, nHandedIn, nStraddle, nStraddleOwned, nStraddleHalo, nNoHalo}; + unsigned long total[12] = {0}; + SU2_MPI::Allreduce(local, total, 12, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); unsigned long localCV = Index_CoarseCV - starting_Index_CoarseCV, totalCV = 0; SU2_MPI::Allreduce(&localCV, &totalCV, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -2644,6 +2977,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long depthMin = 0, depthMax = 0; SU2_MPI::Allreduce(&dmin, &depthMin, 1, MPI_UNSIGNED_LONG, MPI_MIN, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&dmax, &depthMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); + if (depthMin == std::numeric_limits::max()) depthMin = 0; /*--- No fronts anywhere. ---*/ if (rank == MASTER_NODE) { cout << " Paving fronts: " << total[1] << " fronts from " << total[0] << " seed nodes, patch sizes "; @@ -2654,6 +2988,11 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (total[4] + total[5] > 0) cout << "\n Coarse CVs by depth: " << total[5] << " two layers deep, " << total[4] << " one layer (top of a stack)"; + if (total[6] + total[7] > 0) + cout << "\n Stacks handed across partitions: " << total[6] << " sent, " << total[7] << " picked up"; + if (total[8] + total[11] > 0) + cout << "\n Stacks lost at partitions: " << total[8] << " straddled the interface (" << total[9] << " nodes on" + << " this side, " << total[10] << " across), " << total[11] << " blocked with nowhere to hand to"; /*--- Why each front stopped. Reaching a boundary is the one correct stop; everything else is the * mesh failing to offer a layer topologically identical to the current one, broken down by how * it failed. COLLISION and PINCH are two fronts, or two nodes of one front, reaching for the From 8303054ce3e3de60624a0ba092887f7abe5e8f0c Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 4 Sep 2026 15:57:13 +0200 Subject: [PATCH 40/54] fix some more mpi rank and implicit line interaction --- Common/src/geometry/CMultiGridGeometry.cpp | 138 +++++++++++++++------ 1 file changed, 98 insertions(+), 40 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index d858bd2b8e7..d599c394398 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -2374,6 +2374,23 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, vector failReason; vector stopCounts(N_STOP_REASONS, 0); + /*--- Set when a front hands only PART of its footprint over and goes on marching here with what is + * left of it, so the retirement pass at the end of the round knows not to kill it. ---*/ + vector keepLocal; + /*--- The name the handed-over piece travels under. After a split this is NOT the name of the front + * it came from: the two pieces are separate stacks from here on, and giving them one name would + * let the far side group a piece of this stack with a piece of another one. ---*/ + vector handTag; + + /*--- A name for a set of nodes that both ranks sharing them would compute identically. The nodes of + * a footprint are claimed by one front and by no other, so the smallest global index in it is a + * unique name for that front; the +1 leaves 0 free to mean "nothing". ---*/ + auto tagOfSet = [&](const vector& set) { + unsigned long t = std::numeric_limits::max(); + for (auto p : set) t = std::min(t, fine_grid->nodes->GetGlobalIndex(p)); + return t + 1; + }; + /*--- The bid table. Only the index is kept per mesh point, and the bids themselves live in a * compact vector holding one entry per candidate actually bid on this round - a few per front, * against one entry per point in the mesh. Storing a whole CStep per point instead costs about @@ -2389,7 +2406,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, unsigned long nStacks = 0, nSemiCV = 0, nFullCV = 0, nLayers = 0, nCovered = 0; unsigned long nHandedOut = 0, nHandedIn = 0; - unsigned long nStraddle = 0, nStraddleOwned = 0, nStraddleHalo = 0, nNoHalo = 0; + unsigned long nSplit = 0, nSplitLocal = 0, nSplitHanded = 0, nSplitDropped = 0, nNoHalo = 0; /*--- One footprint node arriving from a neighbouring rank, to be regrouped by tag. ---*/ struct CInherited { @@ -2455,6 +2472,28 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, return std::find(pts.begin(), pts.end(), b) != pts.end(); }; + /*--- Is a footprint one connected patch? A set that falls into pieces is not the extrusion of + * anything, so a footprint arriving from a neighbour and a piece left behind by a split both + * have to pass this before they are allowed to carry a stack. ---*/ + auto isConnectedLayer = [&](const vector& layer) { + if (layer.size() < 2) return true; + vector seen(layer.size(), 0); + vector stk{0}; + seen[0] = 1; + size_t nSeen = 1; + while (!stk.empty()) { + const auto cur = stk.back(); + stk.pop_back(); + for (size_t k = 0; k < layer.size(); ++k) { + if (seen[k] || !isAdjacent(layer[cur], layer[k])) continue; + seen[k] = 1; + nSeen++; + stk.push_back(k); + } + } + return nSeen == layer.size(); + }; + /*--- The one test that decides whether a front may advance: is the layer it is about to lay down * TOPOLOGICALLY IDENTICAL to the layer it is standing on? * @@ -2553,6 +2592,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, failed.assign(front.size(), 0); failReason.assign(front.size(), 0); + keepLocal.assign(front.size(), 0); + handTag.assign(front.size(), 0); for (const auto& b : bids) bidIdx[b.node] = NOBID; bids.clear(); bidOwner.clear(); @@ -2679,21 +2720,49 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, prop[f].push_back(s); } - /*--- A layer goes wholly to this rank or wholly to the neighbour. A front that would have to - * split across the interface retires: half a layer is not an extrusion, and stitching the - * two halves back together afterwards would need the two ranks to agree on a footprint - * neither of them holds in full. ---*/ + /*--- A footprint that reaches an interface can be cut by it. If the WHOLE footprint crosses, + * the stack is handed over intact and this front is finished. If only part of it crosses - + * the interface running ALONG the stack rather than across it - the footprint is SPLIT: the + * piece whose successors this rank owns marches on here as a narrower stack, and the rest + * goes to the rank owning the nodes it was reaching for. Both pieces are renamed, because + * they are separate stacks from here on. + * + * Retiring on a cut, which is what this did before, was the expensive part of partitioning: + * an interface that cuts a stack at layer k cuts it at every layer above k as well, so one + * straddle did not cost one layer, it cost the whole remaining column - about a hundred + * nodes each on the flat plate. ---*/ if (failed[f]) { prop[f].clear(); handTo[f].clear(); } else if (!handTo[f].empty()) { - prop[f].clear(); - if (handTo[f].size() != front[f].size()) { - nStraddleOwned += front[f].size() - handTo[f].size(); - nStraddleHalo += handTo[f].size(); - nStraddle++; - handTo[f].clear(); - markFail(f, STOP_PARTITION); + /*--- prop[f] is built in the order of front[f], so this is the piece that stays, in the same + * order, and phi still runs index for index between it and the layer it proposes. ---*/ + vector narrow; + for (const auto& s : prop[f]) narrow.push_back(s.from); + + /*--- A cut can leave the local piece in two disconnected halves - a square footprint cut + * diagonally does exactly that - and that is not a layer. Drop it and hand over the rest; + * the stack still survives on the far side instead of ending here. ---*/ + if (!narrow.empty() && !isConnectedLayer(narrow)) { + nSplitDropped += narrow.size(); + narrow.clear(); + } + + handTag[f] = tagOfSet(handTo[f]); + + if (narrow.empty()) { + prop[f].clear(); + } else { + nSplit++; + nSplitLocal += narrow.size(); + nSplitHanded += handTo[f].size(); + /*--- Close the coarse CV that is open on the WIDE footprint before narrowing, so that no CV + * ever ends up holding two layers of different shape. ---*/ + front[f] = narrow; + emit(f); + nBlock[f] = blockFor(front[f]); + tag[f] = tagOfSet(front[f]); + keepLocal[f] = 1; } } } @@ -2847,8 +2916,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto v = haloVertex[p]; /*--- Two fronts of this rank reaching for the same node: the lower tag takes it, which is * a decision both ranks would reach the same way. ---*/ - if ((tagOut[v] != 0) && (tagOut[v] <= tag[f])) continue; - tagOut[v] = tag[f]; + if ((tagOut[v] != 0) && (tagOut[v] <= handTag[f])) continue; + tagOut[v] = handTag[f]; for (unsigned short d = 0; d < nDim; ++d) dirOut[v * nDim + d] = dirNow[f][d]; } } @@ -2868,13 +2937,15 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } - /*--- Every front that handed over is finished here; the neighbour owns the rest of the stack. ---*/ + /*--- A front that handed its WHOLE footprint over is finished here; the neighbour owns the rest + * of the stack. One that handed over only a piece keeps marching on what was left of it. ---*/ for (unsigned long f = 0; f < front.size(); ++f) { if (handTo[f].empty()) continue; handTo[f].clear(); + nHandedOut++; + if (keepLocal[f]) continue; alive[f] = 0; stopCounts[STOP_PARTITION]++; - nHandedOut++; emit(f); } @@ -2897,23 +2968,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, layer0.push_back(p); } /*--- The footprint has to arrive whole and connected, the same test any other layer passes. ---*/ - if (ok && (layer0.size() > 1)) { - vector seen(layer0.size(), 0); - vector stk{0}; - seen[0] = 1; - size_t nSeen = 1; - while (!stk.empty()) { - const auto cur = stk.back(); - stk.pop_back(); - for (size_t k = 0; k < layer0.size(); ++k) { - if (seen[k] || !isAdjacent(layer0[cur], layer0[k])) continue; - seen[k] = 1; - nSeen++; - stk.push_back(k); - } - } - if (nSeen != layer0.size()) ok = false; - } + if (ok && !isConnectedLayer(layer0)) ok = false; if (ok) { std::array d0{}; @@ -2959,10 +3014,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * stops at the partition, so the number of nodes left to ordinary agglomeration is the number to * watch when adding ranks. Every rank must reach these collectives. ---*/ unsigned long nSeedNodes = seeds.node.size(); - unsigned long local[12] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV, - nHandedOut, nHandedIn, nStraddle, nStraddleOwned, nStraddleHalo, nNoHalo}; - unsigned long total[12] = {0}; - SU2_MPI::Allreduce(local, total, 12, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + unsigned long local[13] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV, nHandedOut, + nHandedIn, nSplit, nSplitLocal, nSplitHanded, nNoHalo, nSplitDropped}; + unsigned long total[13] = {0}; + SU2_MPI::Allreduce(local, total, 13, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); unsigned long localCV = Index_CoarseCV - starting_Index_CoarseCV, totalCV = 0; SU2_MPI::Allreduce(&localCV, &totalCV, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -2990,9 +3045,12 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, << " one layer (top of a stack)"; if (total[6] + total[7] > 0) cout << "\n Stacks handed across partitions: " << total[6] << " sent, " << total[7] << " picked up"; - if (total[8] + total[11] > 0) - cout << "\n Stacks lost at partitions: " << total[8] << " straddled the interface (" << total[9] << " nodes on" - << " this side, " << total[10] << " across), " << total[11] << " blocked with nowhere to hand to"; + if (total[8] > 0) + cout << "\n Footprints split at partitions: " << total[8] << " cut by an interface (" << total[9] + << " nodes marching on here, " << total[10] << " handed across)"; + if (total[11] + total[12] > 0) + cout << "\n Stacks lost at partitions: " << total[11] << " blocked with nowhere to hand to, " << total[12] + << " nodes in split pieces that came apart"; /*--- Why each front stopped. Reaching a boundary is the one correct stop; everything else is the * mesh failing to offer a layer topologically identical to the current one, broken down by how * it failed. COLLISION and PINCH are two fronts, or two nodes of one front, reaching for the From c179fc99bbb9a4b61958d0516d919c7575ab9545 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Fri, 4 Sep 2026 17:37:05 +0200 Subject: [PATCH 41/54] fix 2DZP sym-wall node issue for multilevel MG --- Common/src/geometry/CMultiGridGeometry.cpp | 60 ++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index d599c394398..550c0347ad9 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -835,6 +835,56 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (onPhysBoundary[iFinePoint]) cvOnBoundary[iCoarsePoint] = true; } + /*--- Which physical boundaries each coarse CV sits on, one bit per marker. cvOnBoundary above only + * records THAT a CV touches a boundary, and the repair passes below compare nothing else, so a + * one-child CV on boundary A is free to be merged into a neighbour that lies on boundary B. That + * hands the target CV a marker none of its own children carried: SetVertex gives a coarse CV + * every marker of every child, so the merged CV becomes a vertex of A while its centroid sits + * wherever the B stack put it, and the boundary condition for A is then applied to a control + * volume that is mostly not on A. Comparing the marker SETS - a strictly stronger test than the + * boolean, since an interior CV has an empty set - keeps a merge inside one boundary. ---*/ + vector cvMarkerMask(nPointDomain, 0); + { + vector bitOfMarker(fine_grid->GetnMarker(), -1); + int nBits = 0; + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + if (nBits < 64) bitOfMarker[iMarker] = nBits; + nBits++; + } + if (nBits <= 64) { + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) + for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { + const auto iFinePoint = nodes->GetChildren_CV(iCoarsePoint, iChildren); + for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { + if (bitOfMarker[iMarker] < 0) continue; + if (fine_grid->nodes->GetVertex(iFinePoint, iMarker) >= 0) + cvMarkerMask[iCoarsePoint] |= 1ULL << bitOfMarker[iMarker]; + } + } + } else { + /*--- More markers than bits: fall back to the boolean test. ---*/ + for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) + cvMarkerMask[iCoarsePoint] = cvOnBoundary[iCoarsePoint] ? 1ULL : 0ULL; + } + } + + /*--- A boundary CV built by the paving is the BASE of a stack, and the paving's whole guarantee is + * that every layer above it was given that same footprint. The repair passes must not take the + * footprint away: merging a stack base into the neighbouring stack leaves the column above it + * headless and the merged base wider than either column, so base and first layer no longer line + * up. On the next level that misalignment is fatal rather than cosmetic - the stiffest neighbour + * of the widened base is then a LATERAL one, SeedFrontNodes' hasLayerNormalTo rejects it, the CV + * does not seed a front, and it is swallowed mid-stack by an interior front instead. The coarse + * CV that results carries the boundary marker with its body off the boundary, and the boundary + * condition is applied to it. A one-node patch marching as a one-wide stack is a deliberate + * choice in AgglomerateImplicitLines, not damage for these passes to repair; what they are for + * is the INTERIOR singleton left where a line narrows, and that is untouched by this. ---*/ + auto isStackBase = [&](unsigned long iCoarsePoint) { + return cvOnBoundary[iCoarsePoint] && (iCoarsePoint >= starting_idx_lines_DBG) && + (iCoarsePoint < idx_after_lines_DBG); + }; + vector touchesPartition(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -856,7 +906,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un const auto iCoarsePoint_Complete = nodes->GetPoint(iCoarsePoint, 0); if (mustStayAlone[iCoarsePoint_Complete]) continue; - if (cvOnBoundary[iCoarsePoint] != cvOnBoundary[iCoarsePoint_Complete]) continue; + if (isStackBase(iCoarsePoint) || isStackBase(iCoarsePoint_Complete)) continue; + if (cvMarkerMask[iCoarsePoint] != cvMarkerMask[iCoarsePoint_Complete]) continue; /*--- Check if merging would exceed the maximum agglomeration size ---*/ auto nChildren_Target = nodes->GetnChildren_CV(iCoarsePoint_Complete); @@ -871,7 +922,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint_Complete)) { if (nChildrenToRedistribute == 0) break; if (mustStayAlone[jCoarsePoint]) continue; - if (cvOnBoundary[jCoarsePoint] != cvOnBoundary[iCoarsePoint_Complete]) continue; + if (isStackBase(jCoarsePoint)) continue; + if (cvMarkerMask[jCoarsePoint] != cvMarkerMask[iCoarsePoint_Complete]) continue; auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); if (nChildren_Neighbor < maxAgglomSize) { @@ -951,6 +1003,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; if (mustStayAlone[iCoarsePoint]) continue; + if (isStackBase(iCoarsePoint)) continue; if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the @@ -963,7 +1016,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned short best_nChildren = 0; for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { if (mustStayAlone[jCoarsePoint]) continue; - if (cvOnBoundary[jCoarsePoint] != cvOnBoundary[iCoarsePoint]) continue; + if (isStackBase(jCoarsePoint)) continue; + if (cvMarkerMask[jCoarsePoint] != cvMarkerMask[iCoarsePoint]) continue; const auto nChildren_Neighbor = nodes->GetnChildren_CV(jCoarsePoint); /*--- Skip neighbors already emptied by an earlier merge in this same pass. ---*/ if (nChildren_Neighbor == 0) continue; From 58f90b848738d85e76306357bd88858f95fd77fc Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sat, 5 Sep 2026 14:58:53 +0200 Subject: [PATCH 42/54] cleanup --- Common/include/option_structure.hpp | 7 - Common/src/CConfig.cpp | 15 - Common/src/geometry/CMultiGridGeometry.cpp | 262 +----------------- .../src/integration/CMultiGridIntegration.cpp | 37 +-- config_template.cfg | 13 - 5 files changed, 18 insertions(+), 316 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index cc57de8eab0..c66637ca008 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1134,14 +1134,7 @@ struct CMGOptions { a layer normal to them and may therefore seed paving fronts. It is a SEEDING gate only and never stops a front that has started. See CMultiGridGeometry::SeedFrontNodes. */ - su2double MG_Boundary_Thicken_AR{0.0}; /*!< \brief Local aspect ratio, measured along the boundary normal, at or above - which a boundary coarse CV is left as a flat surface patch instead of - being thickened into the interior. Below it the boundary sits in mesh - that is not stretched normal to itself and the CV is grown inwards to - the full agglomeration size. 0 disables thickening altogether, which is - the historical behaviour. See CMultiGridGeometry::CMultiGridGeometry. */ unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ - su2double MG_Correction_Limit{0.0}; /*!< \brief Max relative change of any solution component from one prolongated FAS correction. 0 = no limit. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ su2double MG_Startup_Convergence{-2.0}; /*!< \brief FMG: orders of magnitude (log10) that CONV_FIELD must drop on the active level before promoting to the next finer one. Negative is a diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 6a802905042..8bc4929260e 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2074,10 +2074,6 @@ void CConfig::SetConfig_Options() { * single-grid runs; MG_COARSE_PREC_FREEZE covers the coarse levels. The fine-grid preconditioner drives the outer * nonlinear convergence, so raise this one with more care. 1 rebuilds every solve. DEFAULT: 1 \ingroup Config*/ addUnsignedLongOption("LINEAR_SOLVER_PREC_FREEZE", Linear_Solver_Prec_Freeze, 1); - /*!\brief MG_CORRECTION_LIMIT\n DESCRIPTION: Largest relative change any solution component may undergo from a single - * prolongated multigrid correction, e.g. 0.1 caps it at 10%. The whole correction vector at a point is scaled by one - * factor so its direction is preserved. 0 disables the limiter (previous behaviour). DEFAULT: 0 \ingroup Config*/ - addDoubleOption("MG_CORRECTION_LIMIT", MGOptions.MG_Correction_Limit, 0.0); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ @@ -2095,17 +2091,6 @@ void CConfig::SetConfig_Options() { * carry a layer normal to them and may therefore seed paving fronts; viscous walls always seed. This is a seeding * gate only and never stops a front that has started. 1.0 lets every boundary seed. DEFAULT: 2.0 \ingroup Config*/ addDoubleOption("MG_IMPLICIT_LINES_MIN_AR", MGOptions.MG_Implicit_Lines_Min_AR, 2.0); - /*!\brief MG_BOUNDARY_THICKEN_AR\n DESCRIPTION: Grow boundary coarse CVs into the interior instead of leaving them as - * surface patches one fine cell thick. The boundary agglomeration can only ever merge points that lie on the boundary - * themselves, so a boundary coarse CV comes out flat: 2x2 nodes on a surface, 2 on a ridge, never the 2x2x2 block the - * interior pass builds. Those CVs are a large share of the coarse grid by count while holding very few nodes each. - * Thickening keeps the surface footprint the boundary agglomeration chose and only adds the layer underneath it, so - * the CV still never straddles two boundary conditions. It is skipped where the mesh carries a stretched layer normal - * to the boundary, i.e. where the local aspect ratio measured along the boundary normal reaches this value, since - * there the flat CV is deliberate semi-coarsening that preserves the wall-normal resolution of a boundary layer. - * Raising it thickens more boundaries, lowering it fewer. 0.0 disables thickening entirely. DEFAULT: 0.0 - * \ingroup Config*/ - addDoubleOption("MG_BOUNDARY_THICKEN_AR", MGOptions.MG_Boundary_Thicken_AR, 0.0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 550c0347ad9..b4c8dbe5e57 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -93,16 +93,19 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * the layers stacked on top of it share one footprint; letting the general scheme claim the wall * first would fix a footprint chosen without any knowledge of the fronts, and the stack above it * could then only be misaligned with its own base. Everything it claims is - * already marked agglomerated, so the boundary and interior passes below simply skip it. ---*/ - const auto starting_idx_lines_DBG = Index_CoarseCV; + * already marked agglomerated, so the boundary and interior passes below simply skip it. + * + * The coarse CVs it creates occupy the half-open index range [firstLineCV, endLineCV), which is + * how the repair passes further down tell them apart from the rest, see isStackBase. ---*/ + const auto firstLineCV = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config); } - const auto idx_after_lines_DBG = Index_CoarseCV; + const auto endLineCV = Index_CoarseCV; /*--- Points carrying a physical boundary condition. SEND_RECEIVE is not one: it only records that - * the point is mirrored on another rank. Used below to tell a genuine interior point, which a - * boundary CV may absorb, from a point on another boundary, which it may not. ---*/ + * the point is mirrored on another rank. Used by the repair passes below to tell which coarse + * CVs touch a boundary, see cvOnBoundary. ---*/ vector onPhysBoundary(fine_grid->GetnPoint(), 0); for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; @@ -115,47 +118,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * it and would otherwise pair up with itself quite happily. ---*/ const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); - vector bmembers; - vector nThicken_DBG(fine_grid->GetnMarker(), 0), nFlat_DBG(fine_grid->GetnMarker(), 0); - - /*--- Whether a boundary coarse CV may be grown into the interior, see MG_BOUNDARY_THICKEN_AR. The - * measurement is only needed when it can be, so a run that leaves the option off pays nothing. ---*/ - const su2double thickenAR = config->GetMGOptions().MG_Boundary_Thicken_AR; - const bool THICKEN = (thickenAR > 0.0); - const CNodeStiffness boundStiff = THICKEN ? ComputeNodeStiffness(fine_grid) : CNodeStiffness(); - - /*--- True where the mesh carries a stretched layer running normal to this boundary, the situation a - * flat boundary CV exists to preserve. Both halves of the test matter, and testing the aspect - * ratio alone is what made a fixed threshold useless on a real mesh: the ratio is undirected, so - * a boundary lying in mesh that is merely graded ALONG itself - a symmetry plane with streamwise - * stretching, say - reads as strongly stretched and never gets thickened, even though nothing - * normal to it would be lost. Requiring the stiffest direction at the node to line up with the - * boundary normal separates the two: only a boundary with cells stacked against it qualifies. ---*/ - constexpr su2double THICKEN_ANGLE_DEG = 20.0; - const su2double thickenCos = cos(THICKEN_ANGLE_DEG * PI_NUMBER / 180.0); - - auto boundaryHasLayer = [&](unsigned long iPoint, unsigned short iMarker) { - const auto jStiffest = boundStiff.jStiffest[iPoint]; - if (jStiffest == std::numeric_limits::max()) return false; - if (boundStiff.AspectRatio(iPoint) < thickenAR) return false; - - const long iVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); - if (iVertex == -1) return false; - su2double normal[MAXNDIM] = {0.0}; - fine_grid->vertex[iMarker][iVertex]->GetNormal(normal); - const su2double nrm = GeometryToolbox::Norm(nDim, normal); - if (nrm <= 0.0) return false; - - su2double vec[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jStiffest), fine_grid->nodes->GetCoord(iPoint), vec); - const su2double len = GeometryToolbox::Norm(nDim, vec); - if (len <= 0.0) return false; - - su2double dot = 0.0; - for (unsigned short d = 0; d < nDim; ++d) dot += (vec[d] / len) * (normal[d] / nrm); - return fabs(dot) >= thickenCos; - }; - /*--- STEP 1: The first step is the boundary agglomeration. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { /*--- Skip periodic boundaries: do not agglomerate on periodic markers. ---*/ @@ -188,8 +150,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- We add the seed point (child) to the parent control volume ---*/ nodes->SetChildren_CV(Index_CoarseCV, 0, iPoint); - bmembers.clear(); - bmembers.push_back(iPoint); bool agglomerate_seed = false; auto counter = 0; unsigned short copy_marker[3] = {}; @@ -311,7 +271,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); nChildren++; - bmembers.push_back(CVPoint); /*--- In 2D, we agglomerate exactly 2 nodes if the nodes are on the line edge. ---*/ if ((nDim == 2) && (counter == 1)) break; /*--- In 3D, we agglomerate exactly 2 nodes if the nodes are on the surface edge. ---*/ @@ -352,60 +311,11 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->SetChildren_CV(Index_CoarseCV, nChildren, CVPoint); nChildren++; - bmembers.push_back(CVPoint); /*--- Apply maxAgglomSize limit for 3D internal boundary face nodes. ---*/ if (nChildren >= maxAgglomSize) break; } } } - - /*--- Thicken the surface patch into the interior. Everything above only ever considers - * candidates that lie on the boundary themselves, because SetBoundAgglomeration refuses - * an interior point outright, so a boundary coarse CV comes out as a film one fine cell - * thick: 2x2 nodes at best on a surface, 2 on a ridge, never the 2x2x2 block the domain - * pass builds everywhere else. That is a coarse CV of 4 nodes where 8 were available, - * and since every boundary of the mesh is covered in them they make up a large share of - * the coarse grid by count while holding very few nodes each. - * - * Growing into the interior fixes that without touching which surface nodes belong - * together: the footprint on the boundary is already decided above, this only adds the - * layer underneath it. Candidates are restricted to genuinely interior points, so the CV - * still cannot straddle two boundary conditions, and the node that joins is the one - * sharing the most faces with what the CV already holds - the same rule STEP 2 uses, - * which is what makes it close into blocks rather than grow into stars. ---*/ - const bool thickenThis = THICKEN && !boundaryHasLayer(iPoint, iMarker); - (thickenThis ? nThicken_DBG : nFlat_DBG)[iMarker]++; - - while (thickenThis && (nChildren < maxAgglomSize)) { - auto best = std::numeric_limits::max(); - unsigned short best_shared = 0; - - for (auto mPoint : bmembers) { - for (auto jPoint : fine_grid->nodes->GetPoints(mPoint)) { - if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; - if (!fine_grid->nodes->GetDomain(jPoint)) continue; - if (onPhysBoundary[jPoint]) continue; - if (!GeometricalCheck(jPoint, fine_grid, config)) continue; - - unsigned short shared = 0; - for (auto kPoint : fine_grid->nodes->GetPoints(jPoint)) - shared += (find(bmembers.begin(), bmembers.end(), kPoint) != bmembers.end()); - - if (shared > best_shared) { - best_shared = shared; - best = jPoint; - } - } - } - - if (best == std::numeric_limits::max()) break; - - fine_grid->nodes->SetParent_CV(best, Index_CoarseCV); - if (fine_grid->nodes->GetAgglomerate_Indirect(best)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); - nodes->SetChildren_CV(Index_CoarseCV, nChildren, best); - nChildren++; - bmembers.push_back(best); - } } /*--- Update the number of children of the coarse control volume. ---*/ @@ -511,9 +421,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un vector frameLen, edgeLen; vector edgeUsed; - const auto idx_after_bound_DBG = Index_CoarseCV; - unsigned long nRejectedSeed_DBG = 0; - auto iteration = 0ul; while (!MGQueue_InnerCV.EmptyQueue() && (iteration < fine_grid->GetnPoint())) { const auto iPoint = MGQueue_InnerCV.NextCV(); @@ -664,13 +571,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- The seed point can not be agglomerated because of size, domain, streching, etc. move the point to the lowest priority ---*/ - nRejectedSeed_DBG++; MGQueue_InnerCV.MoveCV(iPoint, -1); } } - const auto idx_after_domain_DBG = Index_CoarseCV; - /*--- Convert any point that was not agglomerated into a coarse point. ---*/ for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { @@ -683,121 +587,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- TEMPORARY DIAGNOSTIC: size distribution of the coarse CVs, by the phase that created them. - * - * The child count on its own does not say what a CV looks like: eight children can be the 2x2x2 - * block that is wanted, or a 4x2x1 slab, or a 1x8 strip, and in a planar slice through the mesh - * those are indistinguishable from a CV that really is small - a cube shows only its four nodes - * that lie in the slice, a slab edge-on shows two. Counting the fine edges whose two ends fall in - * the same CV separates them without reference to any coordinate direction: a 2x2x2 block has 12 - * internal edges, a flat 2x2 has 4, a 1x4 strip has 3, a pair has 1. So "8 children, 12 edges" is - * a cube and "8 children, 10 edges" is not. ---*/ - vector intEdges_DBG(Index_CoarseCV, 0); - for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { - if (!fine_grid->nodes->GetDomain(iPoint)) continue; - const auto pi = fine_grid->nodes->GetParent_CV(iPoint); - if (pi >= Index_CoarseCV) continue; - for (auto jPoint : fine_grid->nodes->GetPoints(iPoint)) { - if (jPoint <= iPoint) continue; - if (!fine_grid->nodes->GetDomain(jPoint)) continue; - if (fine_grid->nodes->GetParent_CV(jPoint) == pi) intEdges_DBG[pi]++; - } - } - - /*--- Every rank owns a slice of the mesh, so every rank has its own share of these counts. Summing - * them and printing once is the only version that means anything: rank 0's slice is not the - * grid, and printing from all ranks interleaves the lines into each other - at eight ranks the - * output was literally unreadable, with one rank's histogram spliced through the middle of - * another's. Values are packed into one buffer so the whole report costs two collectives. ---*/ - { - constexpr unsigned NPHASE = 5, NPV = 30; /*--- Phases, and values accumulated per phase. ---*/ - /*--- Layout within a phase: [0..9] children histogram, [10] CVs, [11] nodes, [12] cubes, - * [13] slabs, [14] squares, [15] strips, [16..29] internal-edge histogram of the 8s. ---*/ - vector acc(NPHASE * NPV + 5, 0); - - auto histOf = [&](unsigned long lo, unsigned long hi, unsigned phase) { - auto* a = &acc[phase * NPV]; - for (auto c = lo; c < hi; ++c) { - const auto n = nodes->GetnChildren_CV(c); - const auto e = intEdges_DBG[c]; - if (n == 8) { - a[(e >= 12) ? 12 : 13]++; - a[16 + std::min(e, 13)]++; - } - if (n == 4) a[(e >= 4) ? 14 : 15]++; - a[std::min(n, 9)]++; - a[10]++; - a[11] += n; - } - }; - histOf(0, starting_idx_lines_DBG, 0); - histOf(starting_idx_lines_DBG, idx_after_lines_DBG, 1); - histOf(idx_after_lines_DBG, idx_after_bound_DBG, 2); - histOf(idx_after_bound_DBG, idx_after_domain_DBG, 3); - histOf(idx_after_domain_DBG, Index_CoarseCV, 4); - - for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { - if (!fine_grid->nodes->GetDomain(iPoint)) continue; - acc[NPHASE * NPV + 0] += fine_grid->nodes->GetnPoint(iPoint); - acc[NPHASE * NPV + 1]++; - } - acc[NPHASE * NPV + 2] = nRejectedSeed_DBG; - acc[NPHASE * NPV + 3] = iteration; - acc[NPHASE * NPV + 4] = fine_grid->GetnPointDomain(); - - vector tot(acc.size(), 0); - SU2_MPI::Allreduce(acc.data(), tot.data(), static_cast(acc.size()), MPI_UNSIGNED_LONG, MPI_SUM, - SU2_MPI::GetComm()); - - /*--- Thickening is counted per configuration-file marker, not per local marker: ranks agree on - * neither the number nor the order of local markers, because each appends its own - * SEND_RECEIVE markers, so the same index means a different boundary elsewhere. ---*/ - const auto nMarkerCfg = config->GetnMarker_CfgFile(); - vector mk(2 * nMarkerCfg, 0), mkTot(2 * nMarkerCfg, 0); - for (auto m = 0u; m < fine_grid->GetnMarker(); ++m) { - if (config->GetMarker_All_KindBC(m) == SEND_RECEIVE) continue; - const auto c = config->GetMarker_CfgFile_TagBound(config->GetMarker_All_TagBound(m)); - mk[2 * c] += nThicken_DBG[m]; - mk[2 * c + 1] += nFlat_DBG[m]; - } - if (nMarkerCfg > 0) - SU2_MPI::Allreduce(mk.data(), mkTot.data(), static_cast(mk.size()), MPI_UNSIGNED_LONG, MPI_SUM, - SU2_MPI::GetComm()); - - if (rank == MASTER_NODE) { - const char* phaseName[NPHASE] = {"pre-lines ", "implicit lines ", "boundary STEP1 ", "domain STEP2 ", - "leftover single"}; - const auto degN = std::max(tot[NPHASE * NPV + 1], 1ul); - cout << " CV size distribution by phase (maxAgglomSize=" << maxAgglomSize - << ", mean fine-graph degree=" << (su2double(tot[NPHASE * NPV + 0]) / su2double(degN)) - << ", a 2x2x2 block has 12 internal edges):" << endl; - - for (unsigned ph = 0; ph < NPHASE; ++ph) { - const auto* a = &tot[ph * NPV]; - if (a[10] == 0) continue; - cout << " " << phaseName[ph] << ": " << a[10] << " CVs, " << a[11] << " nodes, avg " - << (su2double(a[11]) / su2double(a[10])) << " sizes"; - for (unsigned sz = 1; sz <= 9; ++sz) - if (a[sz] > 0) cout << " " << sz << ":" << a[sz]; - if (a[8] > 0) { - cout << " [of the 8s: " << a[12] << " are 2x2x2 cubes, " << a[13] << " are slabs/strips; internal edges"; - for (unsigned e = 7; e <= 13; ++e) - if (a[16 + e] > 0) cout << " " << (e == 13 ? ">=13" : to_string(e)) << ":" << a[16 + e]; - cout << "]"; - } - if (a[4] > 0) cout << " [of the 4s: " << a[14] << " square, " << a[15] << " strip]"; - cout << endl; - } - cout << " STEP2 rejected seeds: " << tot[NPHASE * NPV + 2] << ", iterations used: " << tot[NPHASE * NPV + 3] - << "/" << tot[NPHASE * NPV + 4] << endl; - for (auto c = 0u; c < nMarkerCfg; ++c) { - if (mkTot[2 * c] + mkTot[2 * c + 1] == 0) continue; - cout << " marker " << config->GetMarker_CfgFile_TagBound(c) << ": thickened " << mkTot[2 * c] - << ", kept flat " << mkTot[2 * c + 1] << endl; - } - } - } - nPointDomain = Index_CoarseCV; nPoint = nPointDomain; @@ -881,8 +670,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * choice in AgglomerateImplicitLines, not damage for these passes to repair; what they are for * is the INTERIOR singleton left where a line narrows, and that is untouched by this. ---*/ auto isStackBase = [&](unsigned long iCoarsePoint) { - return cvOnBoundary[iCoarsePoint] && (iCoarsePoint >= starting_idx_lines_DBG) && - (iCoarsePoint < idx_after_lines_DBG); + return cvOnBoundary[iCoarsePoint] && (iCoarsePoint >= firstLineCV) && (iCoarsePoint < endLineCV); }; vector touchesPartition(nPointDomain, false); @@ -1315,20 +1103,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Optional dump of the finished agglomeration: one line per owned fine point, "x y z parentCV". - * It has to come from the END of the constructor, after both repair passes and the renumbering, - * because those still move fine points between coarse CVs - a dump taken before them shows what - * the agglomeration intended rather than what the solver will actually use. ---*/ - if (getenv("DUMPAGGLOM") != nullptr) { - ofstream fdump(string("agglom_level") + to_string(iMesh) + "_r" + to_string(rank) + ".dat"); - for (auto iPoint = 0ul; iPoint < fine_grid->GetnPoint(); iPoint++) { - if (!fine_grid->nodes->GetDomain(iPoint)) continue; - const auto* c = fine_grid->nodes->GetCoord(iPoint); - fdump << c[0] << " " << c[1] << " " << (nDim == 3 ? c[2] : 0.0) << " " << fine_grid->nodes->GetParent_CV(iPoint) - << "\n"; - } - } - edgeColorGroupSize = config->GetEdgeColoringGroupSize(); } @@ -2082,12 +1856,6 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet nQualified.swap(tmp); } - if ((rank == MASTER_NODE) && (getenv("DUMPAGGLOM") != nullptr)) - for (auto iCfg = 0u; iCfg < nMarkerCfg; ++iCfg) - if (nValid[iCfg] > 0) - cout << " Seed vote " << config->GetMarker_CfgFile_TagBound(iCfg) << ": " << nQualified[iCfg] << "/" - << nValid[iCfg] << " = " << (su2double(nQualified[iCfg]) / su2double(nValid[iCfg])) << endl; - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { if (!canSeed(iMarker)) continue; const auto iCfg = cfgOfMarker[iMarker]; @@ -3049,19 +2817,17 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- How far each front actually got. This is the number to watch when the paved region does not * look like a front: fronts that all reach the same height leave a flat interface with ordinary - * agglomeration, and a spread here is that interface coming out as a staircase instead. ---*/ + * agglomeration, and a spread here is that interface coming out as a staircase instead. + * + * A rank with no fronts of its own must not drag the reported minimum to zero: leaving dmin at + * its sentinel keeps it out of the MPI_MIN below, so the range describes the fronts that exist + * rather than the ranks that have none. ---*/ unsigned long dmin = std::numeric_limits::max(), dmax = 0; for (unsigned long f = 0; f < front.size(); ++f) { if (front[f].empty()) continue; dmin = std::min(dmin, depth[f]); dmax = std::max(dmax, depth[f]); } - /*--- A rank with no fronts of its own must not drag the reported minimum to zero: leaving dmin at - * its sentinel keeps it out of the MPI_MIN below, so the range describes the fronts that exist - * rather than the ranks that have none. ---*/ - if (getenv("DUMPAGGLOM") != nullptr) - cout << " Paving rank " << rank << ": " << front.size() << " fronts, depth " << dmin << ".." << dmax - << ", local nPointDomain " << fine_grid->GetnPointDomain() << endl; /*--- Summary over all ranks. Reporting rank 0's own fronts makes a partitioned run look like a * fraction of the mesh it is not, and hides how much of the layer the partitioning cost: a front diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index ac0a8114a32..c06fb87994d 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -1112,48 +1112,19 @@ void CMultiGridIntegration::SetProlongated_Correction(CSolver *sol_fine, CGeomet /*--- Use the adaptive damping factor uniformly across all prolongation levels. ---*/ const su2double factor = config->GetDamp_Correc_Prolong(); - /*--- Optional cap on how much one coarse-grid correction may move the solution at a point. - * Without it the only guard below is the NaN check, and a correction can drive a cell - * non-physical in a single application - measured on the turbulent flat plate, a W-cycle - * correction cuts wall-adjacent density by 25%, from which the energy equation never - * recovers. The cap is relative and per point, and the whole correction vector is scaled by - * one factor so its direction is preserved (scaling components independently would rotate - * the correction and break the coupling between the equations). - * - * Components that are negligible against the largest one at that point (transverse momentum - * in a freestream cell, say) carry no meaningful relative bound and are skipped, otherwise - * they would veto every correction. ---*/ - const su2double limit = config->GetMGOptions().MG_Correction_Limit; - const bool limiting = (limit > 0.0); - SU2_OMP_FOR_STAT(roundUpDiv(geo_fine->GetnPointDomain(), omp_get_num_threads())) for (auto Point_Fine = 0ul; Point_Fine < geo_fine->GetnPointDomain(); Point_Fine++) { auto* Residual_Fine = sol_fine->LinSysRes.GetBlock(Point_Fine); auto* Solution_Fine = sol_fine->GetNodes()->GetSolution(Point_Fine); - - /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ for (auto iVar = 0u; iVar < nVar; iVar++) { + /*--- Prevent a fine grid divergence due to a coarse grid divergence ---*/ if (Residual_Fine[iVar] != Residual_Fine[iVar]) Residual_Fine[iVar] = 0.0; - } - su2double omega = 1.0; - if (limiting) { - su2double ref = 0.0; - for (auto iVar = 0u; iVar < nVar; iVar++) - ref = max(ref, fabs(Solution_Fine[iVar])); - - for (auto iVar = 0u; iVar < nVar; iVar++) { - const su2double scale = fabs(Solution_Fine[iVar]); - if (scale < 1e-6 * ref) continue; - const su2double correction = fabs(factor * Residual_Fine[iVar]); - if (correction > limit * scale) - omega = min(omega, limit * scale / correction); - } - } + su2double correction = factor * Residual_Fine[iVar]; - for (auto iVar = 0u; iVar < nVar; iVar++) - Solution_Fine[iVar] += omega * factor * Residual_Fine[iVar]; + Solution_Fine[iVar] += correction; + } } END_SU2_OMP_FOR diff --git a/config_template.cfg b/config_template.cfg index 94fd00aa75d..83f7fe3cc4e 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1741,19 +1741,6 @@ MG_IMPLICIT_LINES= NO % so there is normally nothing to cap: 0 means no cap. MG_IMPLICIT_LINES_MAX_LENGTH= 0 % -% Grow boundary coarse CVs into the interior instead of leaving them one fine cell thick. -% The boundary agglomeration can only merge points that lie on the boundary themselves, so a -% boundary coarse CV comes out as a flat surface patch (2x2 nodes on a surface, 2 on a ridge) -% rather than the 2x2x2 block the interior pass builds. Those CVs are a large share of the -% coarse grid by count while holding very few nodes each. Thickening keeps the surface footprint -% and only adds the layer underneath it, so a CV still never straddles two boundary conditions. -% It is skipped where the mesh carries a stretched layer normal to the boundary, i.e. where the -% aspect ratio measured ALONG THE BOUNDARY NORMAL reaches this value: there the flat CV is -% deliberate semi-coarsening that preserves boundary-layer resolution, so viscous walls are left -% alone while symmetry planes and far fields in isotropic mesh are filled out. Raising the value -% thickens more boundaries, lowering it fewer. 0.0 disables thickening (historical behaviour). -MG_BOUNDARY_THICKEN_AR= 0.0 -% % Number of iterations spent on each mesh during the Full Multigrid (FMG) startup phase. After % this many iterations the solution is prolongated to the next finer mesh. It is also the length % of the CFL ramp every level is brought up over, the finest grid included, which is what keeps From 74c7497376dc175794d396d861435517efa43599 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sat, 5 Sep 2026 23:03:59 +0200 Subject: [PATCH 43/54] cleanup --- Common/include/CConfig.hpp | 21 +- Common/src/CConfig.cpp | 20 - Common/src/geometry/CPhysicalGeometry.cpp | 548 +--------------------- Common/src/linear_algebra/CSysSolve.cpp | 15 +- 4 files changed, 16 insertions(+), 588 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index dc218103e6e..3544f028267 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -639,8 +639,7 @@ class CConfig { su2double Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ su2double Deform_Linear_Solver_Error; /*!< \brief Min error of the linear solver for the implicit formulation. */ su2double Linear_Solver_Smoother_Relaxation; /*!< \brief Relaxation factor for iterative linear smoothers. */ - unsigned long Linear_Solver_Iter; - unsigned long Linear_Solver_Prec_Freeze; /*!< \brief Reuse the finest-grid preconditioner for this many solves. */ /*!< \brief Max iterations of the linear solver for the implicit formulation. */ + unsigned long Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ unsigned long Deform_Linear_Solver_Iter; /*!< \brief Max iterations of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Frequency; /*!< \brief Restart frequency of the linear solver for the implicit formulation. */ unsigned long Linear_Solver_Restart_Deflation; /*!< \brief Number of vectors used for deflated restarts. */ @@ -1099,8 +1098,6 @@ class CConfig { su2double ParMETIS_tolerance; /*!< \brief Load balancing tolerance for ParMETIS. */ long ParMETIS_pointWgt; /*!< \brief Load balancing weight given to points. */ long ParMETIS_edgeWgt; /*!< \brief Load balancing weight given to edges. */ - su2double ParMETIS_anisoWgt; /*!< \brief Strength of the anisotropy-aware ParMETIS edge weights. 0 disables them. */ - bool ParMETIS_columnPart; /*!< \brief Partition contracted wall-normal columns instead of individual points. */ unsigned short DirectDiff; /*!< \brief Direct Differentation mode. */ bool DiscreteAdjoint, /*!< \brief AD-based discrete adjoint mode. */ DiscreteAdjointDebug; /*!< \brief Discrete adjoint debug mode using tags. */ @@ -4383,12 +4380,6 @@ class CConfig { */ unsigned long GetLinear_Solver_Iter(void) const { return Linear_Solver_Iter; } - /*! - * \brief Number of consecutive linear solves that reuse one finest-grid preconditioner. - * \return Freeze period, 1 meaning rebuild on every solve. - */ - unsigned long GetLinear_Solver_Prec_Freeze(void) const { return Linear_Solver_Prec_Freeze; } - /*! * \brief Get max number of iterations of the linear solver for the implicit formulation. * \return Max number of iterations of the linear solver for the implicit formulation. @@ -10175,16 +10166,6 @@ class CConfig { */ long GetParMETIS_EdgeWeight() const { return ParMETIS_edgeWgt; } - /*! - * \brief Get the strength of the anisotropy-aware ParMETIS edge weights (0 disables them). - */ - passivedouble GetParMETIS_AnisoWeight() const { return SU2_TYPE::GetValue(ParMETIS_anisoWgt); } - - /*! - * \brief Partition contracted wall-normal columns rather than individual points. - */ - bool GetParMETIS_ColumnPartition() const { return ParMETIS_columnPart; } - /*! * \brief Find the marker index (if any) that is part of a given interface pair. * \param[in] iInterface - Number of the interface pair being tested, starting at 0. diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 8bc4929260e..90a2e889a92 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2069,11 +2069,6 @@ void CConfig::SetConfig_Options() { * (e.g. the ILU factorization) for this many consecutive linear solves instead of rebuilding it every time. * 1 reproduces the previous behaviour exactly. DEFAULT: 1 \ingroup Config*/ addUnsignedLongOption("MG_COARSE_PREC_FREEZE", MGOptions.MG_Coarse_Prec_Freeze, 1); - /*!\brief LINEAR_SOLVER_PREC_FREEZE\n DESCRIPTION: Reuse the linear-solver preconditioner (e.g. the ILU factorization) - * for this many consecutive solves on the finest grid, instead of rebuilding it every time. Applies to MESH_0 and to - * single-grid runs; MG_COARSE_PREC_FREEZE covers the coarse levels. The fine-grid preconditioner drives the outer - * nonlinear convergence, so raise this one with more care. 1 rebuilds every solve. DEFAULT: 1 \ingroup Config*/ - addUnsignedLongOption("LINEAR_SOLVER_PREC_FREEZE", Linear_Solver_Prec_Freeze, 1); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ @@ -3073,21 +3068,6 @@ void CConfig::SetConfig_Options() { /* DESCRIPTION: ParMETIS load balancing weight for edges (equiv. to neighbors) */ addLongOption("PARMETIS_EDGE_WEIGHT", ParMETIS_edgeWgt, 1); - /* DESCRIPTION: Strength of the anisotropy-aware ParMETIS edge weights. ParMETIS is otherwise given no edge weights at - * all, so every edge is equally cheap to cut and partition boundaries slice straight through the stretched cells of a - * boundary layer, splitting the wall-normal columns that implicit-line agglomeration and line-implicit smoothing rely - * on. Weighting an edge by the inverse of its length makes the short wall-normal edges expensive to cut and pushes the - * cuts into the tangential direction instead. On a mesh without stretching all edges are of similar length, the - * weights come out uniform, and the partitioning is the same as with no weights at all. 0 disables the weights. - * DEFAULT: 0 */ - addDoubleOption("PARMETIS_ANISO_WEIGHT", ParMETIS_anisoWgt, 0.0); - /*!\brief PARMETIS_COLUMN_PARTITION\n DESCRIPTION: Contract each wall-normal column of stretched cells into a single - * graph vertex before partitioning, and give every node of a column the colour of its column. A partition boundary - * can then never cross a column, which is what the implicit-line agglomeration and line-implicit smoothing need, and - * the graph ParMETIS actually cuts is the wall surface. Columns are found as connected components of the edges that - * are short at both of their ends, so an isotropic mesh contracts to itself and partitions exactly as before. - * DEFAULT: NO \ingroup Config*/ - addBoolOption("PARMETIS_COLUMN_PARTITION", ParMETIS_columnPart, false); /*--- options that are used in the Hybrid RANS/LES Simulations ---*/ /*!\par CONFIG_CATEGORY:Hybrid_RANSLES Options\ingroup Config*/ diff --git a/Common/src/geometry/CPhysicalGeometry.cpp b/Common/src/geometry/CPhysicalGeometry.cpp index e61df306c34..f7d2d36f198 100644 --- a/Common/src/geometry/CPhysicalGeometry.cpp +++ b/Common/src/geometry/CPhysicalGeometry.cpp @@ -7379,7 +7379,7 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { /*--- Some recommended defaults for the various ParMETIS options. ---*/ - idx_t wgtflag = 2; /*--- Weights on the vertices only, raised to 3 below if edge weights are built. ---*/ + idx_t wgtflag = 2; idx_t numflag = 0; idx_t ncon = 1; real_t ubvec = 1.0 + config->GetParMETIS_Tolerance(); @@ -7412,552 +7412,20 @@ void CPhysicalGeometry::SetColorGrid_Parallel(const CConfig* config) { vwgt[iPoint] = wp + we * (xadj[iPoint + 1] - xadj[iPoint]); } - /*--- Cost of cutting each edge of the graph. - * - * Without these ParMETIS is given no edge weights at all and every edge is equally cheap to - * cut, so nothing stops a partition boundary from running straight through the stretched cells - * of a boundary layer and splitting the wall-normal columns that implicit-line agglomeration - * and line-implicit smoothing are built on. - * - * In a stretched cell the wall-normal spacing is the small one, so the short edges are exactly - * the ones that should stay inside a partition. Weighting an edge by the inverse of its length - * therefore makes cutting across the layer expensive and leaves the long tangential edges as - * the cheap place to cut. Length is used rather than the face area over volume ratio that - * measures the same thing elsewhere, because the dual grid does not exist yet at this point of - * the setup: this runs before SetControlVolume, and only the coordinates are available. - * - * On a mesh with no stretching every edge is of similar length, so the weights come out - * uniform and minimizing their sum is the same problem as minimizing the number of cut edges. - * Such a mesh is therefore partitioned exactly as it is with no weights at all. ---*/ - - vector adjwgt; - const su2double anisoWgt = config->GetParMETIS_AnisoWeight(); - const bool columnPart = config->GetParMETIS_ColumnPartition(); - - const auto firstIdx = pointPartitioner.GetFirstIndexOnRank(rank); - const auto lastIdx = pointPartitioner.GetLastIndexOnRank(rank); - auto isLocal = [&](unsigned long g) { return g >= firstIdx && g < lastIdx; }; - - /*--- Edge lengths and the longest edge at each point. Shared by the edge weights below and by the - * column contraction, both of which need to know how a given edge compares with the ones around - * it. Only the coordinates are available at this point of the setup, the dual grid is built much - * later, so length stands in for the face area over volume ratio used elsewhere. ---*/ - vector edgeLen, maxLen, minLen; - vector isColEdge; - vector> stiffDir; - vector nSend(size, 0), nRecv(size, 0), sDisp(size + 1, 0), rDisp(size + 1, 0); - vector sendIdx, recvIdx; - map remoteMaxLen; - - if ((anisoWgt > 0.0) || columnPart) { - /*--- The graph is split linearly and its entries are global indices, so an edge near a linear - * partition boundary has one end that is not stored here. Those are few, of the order of a - * percent of the entries, and each is asked for from the rank the linear partitioner says - * owns it. The same request lists are reused for every later exchange. ---*/ - vector> wanted(size); - for (auto gPoint : adjacency) - if (!isLocal(gPoint)) wanted[pointPartitioner.GetRankContainingIndex(gPoint)].push_back(gPoint); - - for (int r = 0; r < size; ++r) { - auto& w = wanted[r]; - sort(w.begin(), w.end()); - w.erase(unique(w.begin(), w.end()), w.end()); - nSend[r] = static_cast(w.size()); - } - SU2_MPI::Alltoall(nSend.data(), 1, MPI_INT, nRecv.data(), 1, MPI_INT, comm); - for (int r = 0; r < size; ++r) { - sDisp[r + 1] = sDisp[r] + nSend[r]; - rDisp[r + 1] = rDisp[r] + nRecv[r]; - } - - sendIdx.resize(sDisp[size]); - recvIdx.resize(rDisp[size]); - for (int r = 0; r < size; ++r) copy(wanted[r].begin(), wanted[r].end(), sendIdx.begin() + sDisp[r]); - SU2_MPI::Alltoallv(sendIdx.data(), nSend.data(), sDisp.data(), MPI_UNSIGNED_LONG, recvIdx.data(), nRecv.data(), - rDisp.data(), MPI_UNSIGNED_LONG, comm); - - /*--- Round one, the coordinates of the points that were asked for. ---*/ - map> remoteCoord; - { - vector sendBuf(static_cast(rDisp[size]) * nDim), - recvBuf(static_cast(sDisp[size]) * nDim); - for (size_t i = 0; i < recvIdx.size(); ++i) - for (unsigned short iDim = 0; iDim < nDim; ++iDim) - sendBuf[i * nDim + iDim] = nodes->GetCoord(recvIdx[i] - firstIdx, iDim); - - vector nS(size), nR(size), sD(size), rD(size); - for (int r = 0; r < size; ++r) { - nS[r] = nRecv[r] * nDim; - nR[r] = nSend[r] * nDim; - sD[r] = rDisp[r] * nDim; - rD[r] = sDisp[r] * nDim; - } - SU2_MPI::Alltoallv(sendBuf.data(), nS.data(), sD.data(), MPI_DOUBLE, recvBuf.data(), nR.data(), rD.data(), - MPI_DOUBLE, comm); - for (size_t i = 0; i < sendIdx.size(); ++i) { - array c = {0.0, 0.0, 0.0}; - for (unsigned short iDim = 0; iDim < nDim; ++iDim) c[iDim] = recvBuf[i * nDim + iDim]; - remoteCoord[sendIdx[i]] = c; - } - } - - edgeLen.assign(adjacency.size(), 0.0); - maxLen.assign(nPoint, 0.0); - minLen.assign(nPoint, std::numeric_limits::max()); - stiffDir.assign(nPoint, {0.0, 0.0, 0.0}); - - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { - const auto gPoint = adjacency[k]; - array tmp = {0.0, 0.0, 0.0}; - const su2double* coord_j = nullptr; - if (isLocal(gPoint)) { - coord_j = nodes->GetCoord(gPoint - firstIdx); - } else { - const auto it = remoteCoord.find(gPoint); - if (it == remoteCoord.end()) continue; - tmp = it->second; - coord_j = tmp.data(); - } - edgeLen[k] = GeometryToolbox::Distance(nDim, nodes->GetCoord(iPoint), coord_j); - maxLen[iPoint] = max(maxLen[iPoint], edgeLen[k]); - - /*--- The shortest edge at a node fixes the direction its column runs in. ---*/ - if (edgeLen[k] < minLen[iPoint]) { - minLen[iPoint] = edgeLen[k]; - for (unsigned short iDim = 0; iDim < nDim; ++iDim) - stiffDir[iPoint][iDim] = (coord_j[iDim] - nodes->GetCoord(iPoint, iDim)) / edgeLen[k]; - } - } - if (minLen[iPoint] == std::numeric_limits::max()) minLen[iPoint] = 0.0; - } - - /*--- Round two, the longest edge at each of the remote points. ---*/ - { - vector sendBuf(rDisp[size]), recvBuf(sDisp[size]); - for (size_t i = 0; i < recvIdx.size(); ++i) sendBuf[i] = maxLen[recvIdx[i] - firstIdx]; - SU2_MPI::Alltoallv(sendBuf.data(), nRecv.data(), rDisp.data(), MPI_DOUBLE, recvBuf.data(), nSend.data(), - sDisp.data(), MPI_DOUBLE, comm); - for (size_t i = 0; i < sendIdx.size(); ++i) remoteMaxLen[sendIdx[i]] = recvBuf[i]; - } - - /*--- Which edges make up the wall-normal columns. - * - * An edge qualifies when it is short compared with the longest edge at both of its ends, so - * the mesh is stretched there, AND it runs along the shortest edge at both of its ends. The - * second test is what keeps a column one-dimensional: in a cell graded in two directions at - * once, several directions are shorter than the longest one and testing only the length links - * the columns sideways into sheets, which then contract into a handful of enormous vertices - * instead of one per wall node. Requiring alignment with the stiffest direction of both ends - * leaves exactly the chains that run through the layer. ---*/ - if (columnPart) { - map> remoteDir; - { - vector sendBuf(static_cast(rDisp[size]) * nDim), - recvBuf(static_cast(sDisp[size]) * nDim); - for (size_t i = 0; i < recvIdx.size(); ++i) - for (unsigned short iDim = 0; iDim < nDim; ++iDim) - sendBuf[i * nDim + iDim] = stiffDir[recvIdx[i] - firstIdx][iDim]; - vector nS(size), nR(size), sD(size), rD(size); - for (int r = 0; r < size; ++r) { - nS[r] = nRecv[r] * nDim; - nR[r] = nSend[r] * nDim; - sD[r] = rDisp[r] * nDim; - rD[r] = sDisp[r] * nDim; - } - SU2_MPI::Alltoallv(sendBuf.data(), nS.data(), sD.data(), MPI_DOUBLE, recvBuf.data(), nR.data(), rD.data(), - MPI_DOUBLE, comm); - for (size_t i = 0; i < sendIdx.size(); ++i) { - array d = {0.0, 0.0, 0.0}; - for (unsigned short iDim = 0; iDim < nDim; ++iDim) d[iDim] = recvBuf[i * nDim + iDim]; - remoteDir[sendIdx[i]] = d; - } - } - - const su2double COLUMN_FRACTION = 0.5; /*!< Stretched enough for the edge to be part of a column. */ - const su2double COLUMN_COS = 0.9; /*!< Aligned enough with the stiffest direction at both ends. */ - - isColEdge.assign(adjacency.size(), 0); - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { - if (edgeLen[k] <= 0.0) continue; - const auto gPoint = adjacency[k]; - - su2double maxLen_j = 0.0; - array dir_j = {0.0, 0.0, 0.0}, coord_j = {0.0, 0.0, 0.0}; - if (isLocal(gPoint)) { - maxLen_j = maxLen[gPoint - firstIdx]; - dir_j = stiffDir[gPoint - firstIdx]; - for (unsigned short iDim = 0; iDim < nDim; ++iDim) coord_j[iDim] = nodes->GetCoord(gPoint - firstIdx, iDim); - } else { - const auto itM = remoteMaxLen.find(gPoint); - const auto itD = remoteDir.find(gPoint); - const auto itC = remoteCoord.find(gPoint); - if ((itM == remoteMaxLen.end()) || (itD == remoteDir.end()) || (itC == remoteCoord.end())) continue; - maxLen_j = itM->second; - dir_j = itD->second; - coord_j = itC->second; - } - - if (edgeLen[k] > COLUMN_FRACTION * maxLen[iPoint]) continue; - if (edgeLen[k] > COLUMN_FRACTION * maxLen_j) continue; - - su2double e[3] = {0.0, 0.0, 0.0}, di = 0.0, dj = 0.0; - for (unsigned short iDim = 0; iDim < nDim; ++iDim) { - e[iDim] = (coord_j[iDim] - nodes->GetCoord(iPoint, iDim)) / edgeLen[k]; - di += e[iDim] * stiffDir[iPoint][iDim]; - dj += e[iDim] * dir_j[iDim]; - } - if ((fabs(di) < COLUMN_COS) || (fabs(dj) < COLUMN_COS)) continue; - - isColEdge[k] = 1; - } - } - } - } - - /*--- Cost of cutting each edge of the graph. - * - * Without these ParMETIS is given no edge weights at all and every edge is equally cheap to - * cut, so nothing stops a partition boundary from running straight through the stretched cells - * of a boundary layer and splitting the wall-normal columns that implicit-line agglomeration - * and line-implicit smoothing are built on. - * - * An edge is expensive to cut when it is much shorter than the other edges meeting it, which - * is the definition of the local cell aspect ratio and is exactly the situation inside a - * boundary layer, where the short edges are the wall-normal ones. Comparing an edge only - * against its own neighbourhood, rather than against a global length, is what keeps the - * measure a ratio: scaling the whole mesh, or refining one region of it isotropically, leaves - * every weight unchanged. Averaging the two ends keeps the weight of an edge symmetric, which - * ParMETIS requires. - * - * Note this only discourages such cuts, it cannot forbid them, and by making the wall-normal - * direction expensive it pushes every cut into the wall surface instead, which splits the wall - * faces that tangential bundling needs. PARMETIS_COLUMN_PARTITION below removes the choice - * rather than reweighting it. ---*/ - - if (anisoWgt > 0.0) { - const idx_t MAX_EDGE_WEIGHT = 1000; - adjwgt.resize(adjacency.size(), 1); - - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { - if (edgeLen[k] <= 0.0) continue; - const auto gPoint = adjacency[k]; - su2double maxLen_j = 0.0; - if (isLocal(gPoint)) { - maxLen_j = maxLen[gPoint - firstIdx]; - } else { - const auto it = remoteMaxLen.find(gPoint); - if (it == remoteMaxLen.end()) continue; - maxLen_j = it->second; - } - const su2double ratio = 0.5 * (maxLen[iPoint] + maxLen_j) / edgeLen[k]; - const su2double w = 1.0 + anisoWgt * (ratio - 1.0); - adjwgt[k] = static_cast(min(max(w, 1.0), MAX_EDGE_WEIGHT)); - } - } - wgtflag = 3; /*--- Weights on both the vertices and the edges. ---*/ - } - /*--- Create some structures that ParMETIS needs to output the partitioning. ---*/ idx_t edgecut; vector part(nPoint); - bool coloured = false; - - /*================================================================================================== - * Column-contracted partitioning. - * - * Weighting edges can only make a cut across a boundary-layer column unattractive; the partitioner - * is still free to make it, and by pricing the wall-normal direction out it takes its cuts through - * the wall surface instead, which splits the wall faces that the tangential bundling of implicit - * lines is built from. Contracting removes the choice: each column of stretched cells becomes one - * vertex of the graph handed to ParMETIS, so no cut can pass through a column, and the graph that - * is actually cut is the wall surface with the layer hanging off it. - * - * A column is a connected component of the edges that are short at BOTH ends. At a wall node of a - * stretched cell the wall-normal edge is a fraction of the tangential ones, so it qualifies while - * the tangential ones do not, and the components come out as the wall-normal chains. Where the - * mesh is isotropic every edge is about as long as its neighbours, nothing qualifies, each point - * is its own column, and the contracted graph is the original one - such a mesh is partitioned - * exactly as it would have been. Nothing here needs to know where the walls are. - *================================================================================================*/ - - if (columnPart) { - constexpr unsigned long MAX_SWEEPS = 500; - - /*--- Label propagation: every point starts as its own column and repeatedly takes the smallest - * label across a column edge, so a column ends up named after its lowest global index. The - * number of sweeps needed is the length of the longest column, not the size of the mesh. ---*/ - vector label(nPoint); - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) label[iPoint] = firstIdx + iPoint; - - map remoteLabel; - for (auto g : sendIdx) remoteLabel[g] = g; - - unsigned long nSweeps = 0; - for (int changed = 1; changed != 0;) { - changed = 0; - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { - if (!isColEdge[k]) continue; - const auto gPoint = adjacency[k]; - unsigned long lj; - if (isLocal(gPoint)) { - lj = label[gPoint - firstIdx]; - } else { - const auto it = remoteLabel.find(gPoint); - if (it == remoteLabel.end()) continue; - lj = it->second; - } - if (lj < label[iPoint]) { - label[iPoint] = lj; - changed = 1; - } - } - } - - /*--- Refresh the labels of the points that are not stored here. ---*/ - { - vector sendBuf(rDisp[size]), recvBuf(sDisp[size]); - for (size_t i = 0; i < recvIdx.size(); ++i) sendBuf[i] = label[recvIdx[i] - firstIdx]; - SU2_MPI::Alltoallv(sendBuf.data(), nRecv.data(), rDisp.data(), MPI_UNSIGNED_LONG, recvBuf.data(), nSend.data(), - sDisp.data(), MPI_UNSIGNED_LONG, comm); - for (size_t i = 0; i < sendIdx.size(); ++i) { - if (recvBuf[i] != remoteLabel[sendIdx[i]]) changed = 1; - remoteLabel[sendIdx[i]] = recvBuf[i]; - } - } - - int global_changed = 0; - SU2_MPI::Allreduce(&changed, &global_changed, 1, MPI_INT, MPI_MAX, comm); - changed = global_changed; - - if (++nSweeps >= MAX_SWEEPS) break; - } - - /*--- A column is owned by whichever rank the linear partitioner says holds the index it is named - * after, which is one of its own points, so every column has exactly one owner. Number the - * columns consecutively, owner by owner, to give ParMETIS the contiguous distribution it - * expects. ---*/ - unsigned long nColLocal = 0; - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) - if (label[iPoint] == firstIdx + iPoint) nColLocal++; - - vector colCount(size, 0); - SU2_MPI::Allgather(&nColLocal, 1, MPI_UNSIGNED_LONG, colCount.data(), 1, MPI_UNSIGNED_LONG, comm); - - vector cvtxdist(size + 1, 0); - for (int r = 0; r < size; ++r) cvtxdist[r + 1] = cvtxdist[r] + static_cast(colCount[r]); - const auto nColGlobal = static_cast(cvtxdist[size]); - - map colIndexOfRoot; /*!< Global point index of a root -> column index. */ - { - unsigned long next = static_cast(cvtxdist[rank]); - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) - if (label[iPoint] == firstIdx + iPoint) colIndexOfRoot[firstIdx + iPoint] = next++; - } - - /*--- Ask the owner of each label for the column index it was given. ---*/ - auto askOwners = [&](const vector& keys, vector& values) { - vector> want(size); - for (auto key : keys) want[pointPartitioner.GetRankContainingIndex(key)].push_back(key); - vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); - for (int r = 0; r < size; ++r) { - sort(want[r].begin(), want[r].end()); - want[r].erase(unique(want[r].begin(), want[r].end()), want[r].end()); - nS[r] = static_cast(want[r].size()); - } - SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); - for (int r = 0; r < size; ++r) { - sD[r + 1] = sD[r] + nS[r]; - rD[r + 1] = rD[r] + nR[r]; - } - vector qs(sD[size]), qr(rD[size]); - for (int r = 0; r < size; ++r) copy(want[r].begin(), want[r].end(), qs.begin() + sD[r]); - SU2_MPI::Alltoallv(qs.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, qr.data(), nR.data(), rD.data(), - MPI_UNSIGNED_LONG, comm); - - vector as(rD[size]), ar(sD[size]); - for (size_t i = 0; i < qr.size(); ++i) { - const auto it = colIndexOfRoot.find(qr[i]); - as[i] = (it == colIndexOfRoot.end()) ? nColGlobal : it->second; - } - SU2_MPI::Alltoallv(as.data(), nR.data(), rD.data(), MPI_UNSIGNED_LONG, ar.data(), nS.data(), sD.data(), - MPI_UNSIGNED_LONG, comm); - - map answer; - for (int r = 0; r < size; ++r) - for (int i = sD[r]; i < sD[r + 1]; ++i) answer[qs[i]] = ar[i]; - - values.resize(keys.size()); - for (size_t i = 0; i < keys.size(); ++i) { - const auto it = answer.find(keys[i]); - values[i] = (it == answer.end()) ? nColGlobal : it->second; - } - }; - - /*--- Column index of every point stored here, and of every point across a linear boundary. ---*/ - vector myKeys(nPoint); - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) myKeys[iPoint] = label[iPoint]; - vector colOfPoint; - askOwners(myKeys, colOfPoint); - - vector remoteKeys; - remoteKeys.reserve(sendIdx.size()); - for (auto g : sendIdx) remoteKeys.push_back(remoteLabel[g]); - vector colOfRemoteVal; - askOwners(remoteKeys, colOfRemoteVal); - map colOfRemote; - for (size_t i = 0; i < sendIdx.size(); ++i) colOfRemote[sendIdx[i]] = colOfRemoteVal[i]; - - /*--- Send every contracted edge, and every point's weight, to the rank owning the column. ---*/ - vector> outEdge(size), outWeight(size); - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - const auto cA = colOfPoint[iPoint]; - if (cA >= nColGlobal) continue; - int owner = 0; - while ((owner + 1 < size) && (static_cast(cA) >= cvtxdist[owner + 1])) owner++; - outWeight[owner].push_back(cA); - - for (auto k = xadj[iPoint]; k < xadj[iPoint + 1]; ++k) { - const auto gPoint = adjacency[k]; - unsigned long cB; - if (isLocal(gPoint)) { - cB = colOfPoint[gPoint - firstIdx]; - } else { - const auto it = colOfRemote.find(gPoint); - if (it == colOfRemote.end()) continue; - cB = it->second; - } - if ((cB >= nColGlobal) || (cB == cA)) continue; - outEdge[owner].push_back(cA); - outEdge[owner].push_back(cB); - } - } - - auto shipPairs = [&](vector>& out, vector& in) { - vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); - for (int r = 0; r < size; ++r) nS[r] = static_cast(out[r].size()); - SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); - for (int r = 0; r < size; ++r) { - sD[r + 1] = sD[r] + nS[r]; - rD[r + 1] = rD[r] + nR[r]; - } - vector sb(sD[size]); - in.resize(rD[size]); - for (int r = 0; r < size; ++r) copy(out[r].begin(), out[r].end(), sb.begin() + sD[r]); - SU2_MPI::Alltoallv(sb.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, in.data(), nR.data(), rD.data(), - MPI_UNSIGNED_LONG, comm); - }; - - vector inEdge, inWeight; - shipPairs(outEdge, inEdge); - shipPairs(outWeight, inWeight); - - /*--- Assemble the contracted graph for the columns owned here. ---*/ - const auto myFirstCol = static_cast(cvtxdist[rank]); - const auto nColMine = static_cast(cvtxdist[rank + 1] - cvtxdist[rank]); - - vector> cadj(nColMine); - for (size_t i = 0; i + 1 < inEdge.size(); i += 2) { - const auto a = inEdge[i] - myFirstCol; - if (a < nColMine) cadj[a].push_back(inEdge[i + 1]); - } - for (auto& v : cadj) { - sort(v.begin(), v.end()); - v.erase(unique(v.begin(), v.end()), v.end()); - } - - vector cvwgt(nColMine, 0); - for (auto c : inWeight) - if (c - myFirstCol < nColMine) cvwgt[c - myFirstCol]++; - for (auto& w : cvwgt) w = max(w, 1); - - vector cxadj(nColMine + 1, 0), cadjncy; - for (unsigned long c = 0; c < nColMine; ++c) cxadj[c + 1] = cxadj[c] + static_cast(cadj[c].size()); - cadjncy.reserve(cxadj[nColMine]); - for (auto& v : cadj) - for (auto b : v) cadjncy.push_back(static_cast(b)); - - /*--- A column with no neighbours at all would make ParMETIS fail; that only happens on a mesh of - * one point per rank, but check rather than risk it. ---*/ - unsigned long nEdgeLocal = cadjncy.size(), nEdgeGlobal = 0; - SU2_MPI::Allreduce(&nEdgeLocal, &nEdgeGlobal, 1, MPI_UNSIGNED_LONG, MPI_SUM, comm); - - if (nEdgeGlobal > 0) { - idx_t cwgtflag = 2, cedgecut = 0; - vector cpart(max(nColMine, 1)); - - if (rank == MASTER_NODE) { - cout << "Calling ParMETIS on " << nColGlobal << " contracted columns (" << Global_nPointDomain << " points, " - << nSweeps << " sweeps)..."; - } - auto cerr_ = ParMETIS_V3_PartKway(cvtxdist.data(), cxadj.data(), cadjncy.data(), cvwgt.data(), nullptr, - &cwgtflag, &numflag, &ncon, &nparts, tpwgts.data(), &ubvec, options, &cedgecut, - cpart.data(), &comm); - if (cerr_ != METIS_OK) SU2_MPI::Error("Column partitioning failed.", CURRENT_FUNCTION); - if (rank == MASTER_NODE) cout << " complete (" << cedgecut << " column cuts)." << endl; - - /*--- Give every point the colour of its column. ---*/ - map myColColour; - for (unsigned long c = 0; c < nColMine; ++c) myColColour[myFirstCol + c] = static_cast(cpart[c]); - - vector> want(size); - for (auto c : colOfPoint) { - if (c >= nColGlobal) continue; - int owner = 0; - while ((owner + 1 < size) && (static_cast(c) >= cvtxdist[owner + 1])) owner++; - want[owner].push_back(c); - } - vector nS(size, 0), nR(size, 0), sD(size + 1, 0), rD(size + 1, 0); - for (int r = 0; r < size; ++r) { - sort(want[r].begin(), want[r].end()); - want[r].erase(unique(want[r].begin(), want[r].end()), want[r].end()); - nS[r] = static_cast(want[r].size()); - } - SU2_MPI::Alltoall(nS.data(), 1, MPI_INT, nR.data(), 1, MPI_INT, comm); - for (int r = 0; r < size; ++r) { - sD[r + 1] = sD[r] + nS[r]; - rD[r + 1] = rD[r] + nR[r]; - } - vector qs(sD[size]), qr(rD[size]); - for (int r = 0; r < size; ++r) copy(want[r].begin(), want[r].end(), qs.begin() + sD[r]); - SU2_MPI::Alltoallv(qs.data(), nS.data(), sD.data(), MPI_UNSIGNED_LONG, qr.data(), nR.data(), rD.data(), - MPI_UNSIGNED_LONG, comm); - vector as(rD[size]), ar(sD[size]); - for (size_t i = 0; i < qr.size(); ++i) { - const auto it = myColColour.find(qr[i]); - as[i] = (it == myColColour.end()) ? 0 : it->second; - } - SU2_MPI::Alltoallv(as.data(), nR.data(), rD.data(), MPI_UNSIGNED_LONG, ar.data(), nS.data(), sD.data(), - MPI_UNSIGNED_LONG, comm); - map colour; - for (int r = 0; r < size; ++r) - for (int i = sD[r]; i < sD[r + 1]; ++i) colour[qs[i]] = ar[i]; - - for (unsigned long iPoint = 0; iPoint < nPoint; ++iPoint) { - const auto it = colour.find(colOfPoint[iPoint]); - part[iPoint] = (it == colour.end()) ? 0 : static_cast(it->second); - } - coloured = true; - } else if (rank == MASTER_NODE) { - cout << "Column partitioning found no graph to cut, falling back to point partitioning." << endl; - } - } /*--- Calling ParMETIS ---*/ - if (!coloured) { - if (rank == MASTER_NODE) cout << "Calling ParMETIS..."; - auto err = ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), - adjwgt.empty() ? nullptr : adjwgt.data(), &wgtflag, &numflag, &ncon, &nparts, - tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); - if (err != METIS_OK) SU2_MPI::Error("Partitioning failed.", CURRENT_FUNCTION); - if (rank == MASTER_NODE) { - cout << " graph partitioning complete (" << edgecut << " edge cuts)." << endl; - } + if (rank == MASTER_NODE) cout << "Calling ParMETIS..."; + auto err = + ParMETIS_V3_PartKway(vtxdist.data(), xadj.data(), adjacency.data(), vwgt.data(), nullptr, &wgtflag, &numflag, + &ncon, &nparts, tpwgts.data(), &ubvec, options, &edgecut, part.data(), &comm); + if (err != METIS_OK) SU2_MPI::Error("Partitioning failed.", CURRENT_FUNCTION); + if (rank == MASTER_NODE) { + cout << " graph partitioning complete (" << edgecut << " edge cuts)." << endl; } /*--- Store the results of the partitioning (note that this is local diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index aa6ca89515b..5f20fbc6a5b 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -1445,20 +1445,19 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con * Build() is all that is needed to reuse it. * * Restricted to the standard solver mode (mesh deformation and gradient smoothing are left - * alone). Coarse levels and the finest grid have separate periods because the finest-grid - * preconditioner drives the outer nonlinear convergence and so carries more risk. The first - * solve on this instance always builds (count 0), which matters because the factorization is - * otherwise uninitialized. + * alone), and to the coarse levels: the finest grid always rebuilds, because its preconditioner + * drives the outer nonlinear convergence and freezing it measured worth far less than freezing + * the coarse levels. The first solve on this instance always builds (count 0), which matters + * because the factorization is otherwise uninitialized. * * The decision is taken by one thread and read by all of them, because Build() is internally * OpenMP-parallel and every thread must make the same choice. ---*/ BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { unsigned long freeze = 1; - if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr) { - freeze = (geometry->GetMGLevel() != MESH_0) ? config->GetMGOptions().MG_Coarse_Prec_Freeze - : config->GetLinear_Solver_Prec_Freeze(); - freeze = std::max(1, freeze); + if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && + geometry->GetMGLevel() != MESH_0) { + freeze = std::max(1, config->GetMGOptions().MG_Coarse_Prec_Freeze); } buildPrecThisSolve = (precSolveCount % freeze == 0); precSolveCount++; From 2572fe6d34028f789481712c3ed720184b9d7404 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 00:03:23 +0200 Subject: [PATCH 44/54] precommit --- Common/include/geometry/CMultiGridGeometry.hpp | 1 - Common/include/option_structure.hpp | 1 - Common/src/CConfig.cpp | 4 ---- Common/src/geometry/CMultiGridGeometry.cpp | 8 +------- Common/src/linear_algebra/CSysSolve.cpp | 3 +-- config_template.cfg | 5 ----- 6 files changed, 2 insertions(+), 20 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index 601fb891d69..beaafee5fbe 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -138,7 +138,6 @@ class CMultiGridGeometry final : public CGeometry { STOP_NO_NEIGHBOR, /*!< \brief A front node had no free neighbour left to step onto. */ STOP_TOPOLOGY, /*!< \brief The next layer was not isomorphic to the current one. */ STOP_GEOMETRY, /*!< \brief A node of the next layer failed GeometricalCheck. */ - STOP_MAX_LENGTH, /*!< \brief Hit the MG_IMPLICIT_LINES_MAX_LENGTH safety cap, off by default. */ N_STOP_REASONS }; diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index c66637ca008..d48261ce391 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1125,7 +1125,6 @@ struct CMGOptions { bool MG_Smooth_Output{false}; /*!< \brief Output compact per-cycle smoothing summary. */ su2double MG_Smooth_StagnationTol{0.0}; /*!< \brief Stagnation early exit: stop if current_rms >= prev_rms * tol. 0 = disabled. */ bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ - unsigned long MG_Implicit_Lines_MaxLength{0}; /*!< \brief Safety cap on paving stack depth in layers, 0 for none. */ unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse CV tangential to the wall. 0 = dimension-appropriate default (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 90a2e889a92..4cc52a2ff79 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2073,10 +2073,6 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); - /*!\brief MG_IMPLICIT_LINES_MAX_LENGTH\n DESCRIPTION: Safety cap on how many layers deep a paving front - * may go. A front is meant to run until it reaches a boundary or the mesh stops offering a layer - * topologically identical to the one below it, so this is off by default. DEFAULT: 0 (no cap) \ingroup Config*/ - addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_LENGTH", MGOptions.MG_Implicit_Lines_MaxLength, 0); /*!\brief MG_IMPLICIT_LINES_MAX_GROUP\n DESCRIPTION: Maximum number of parallel implicit lines merged tangential to * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index b4c8dbe5e57..ac8b4280a15 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -2103,7 +2103,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- Safety cap on stack depth, 0 for none. A front is meant to run until it reaches a boundary or * the mesh stops offering a clean extrusion, so this is off by default. ---*/ - const unsigned long MAX_LINE_LENGTH = config->GetMGOptions().MG_Implicit_Lines_MaxLength; unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; @@ -2428,11 +2427,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, prop[f].clear(); handTo[f].clear(); - if ((MAX_LINE_LENGTH > 0) && (depth[f] + 1 >= MAX_LINE_LENGTH)) { - markFail(f, STOP_MAX_LENGTH); - continue; - } - for (auto n : front[f]) { auto best = NO_POINT; su2double best_dot = -2.0, best_len = 0.0, best_dir[MAXNDIM] = {0.0}; @@ -2879,7 +2873,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, << stopTotal[STOP_PARTITION] << ", front-collision " << stopTotal[STOP_COLLISION] << ", pinch " << stopTotal[STOP_PINCH] << ", already-agglomerated " << stopTotal[STOP_AGGLOMERATED] << ", dead-end " << stopTotal[STOP_NO_NEIGHBOR] << ", topology " << stopTotal[STOP_TOPOLOGY] << ", geometry " - << stopTotal[STOP_GEOMETRY] << ", max-length " << stopTotal[STOP_MAX_LENGTH]; + << stopTotal[STOP_GEOMETRY]; cout << endl; } } diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index 5f20fbc6a5b..236dce58271 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -1455,8 +1455,7 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { unsigned long freeze = 1; - if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && - geometry->GetMGLevel() != MESH_0) { + if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && geometry->GetMGLevel() != MESH_0) { freeze = std::max(1, config->GetMGOptions().MG_Coarse_Prec_Freeze); } buildPrecThisSolve = (precSolveCount % freeze == 0); diff --git a/config_template.cfg b/config_template.cfg index 83f7fe3cc4e..e5fdd436aba 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1736,11 +1736,6 @@ MG_MIN_MESHSIZE= 500 % Enable agglomeration along implicit lines seeded from viscous walls (NO, YES) MG_IMPLICIT_LINES= NO % -% Safety cap on how many layers deep a paving front may go. A front runs until it reaches a -% boundary or until the mesh stops offering a layer topologically identical to the one below it, -% so there is normally nothing to cap: 0 means no cap. -MG_IMPLICIT_LINES_MAX_LENGTH= 0 -% % Number of iterations spent on each mesh during the Full Multigrid (FMG) startup phase. After % this many iterations the solution is prolongated to the next finer mesh. It is also the length % of the CFL ramp every level is brought up over, the finest grid included, which is what keeps From 5eb8d9ba9e55fc8b690e269624726b04dd2bceff Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 09:21:31 +0200 Subject: [PATCH 45/54] fix passivedouble --- Common/src/geometry/CMultiGridGeometry.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index ac8b4280a15..948284682b0 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1747,8 +1747,8 @@ bool VertexUnitNormal(const CGeometry* grid, unsigned short nDim, unsigned long CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeometry* fine_grid, const CConfig* config, const CNodeStiffness& stiff) const { /*--- Fraction of a marker's nodes that must sit in a layer before the whole marker may seed. ---*/ - constexpr su2double QUALIFIED_FRACTION = 0.5; - constexpr su2double ANGLE_THRESHOLD_DEG = 30.0; + constexpr passivedouble QUALIFIED_FRACTION = 0.5; + constexpr passivedouble ANGLE_THRESHOLD_DEG = 30.0; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); const su2double MIN_AR = config->GetMGOptions().MG_Implicit_Lines_Min_AR; @@ -2095,14 +2095,14 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- How nearly parallel a step must be to a boundary's normal to count as running INTO that * boundary rather than along it. This is not a limit on where a front may go - it is only how * "the front has reached a boundary" is recognised. ---*/ - constexpr su2double BOUNDARY_ALIGN_DEG = 30.0; + constexpr passivedouble BOUNDARY_ALIGN_DEG = 30.0; const su2double cos_boundary = cos(BOUNDARY_ALIGN_DEG * PI_NUMBER / 180.0); /*--- Weight of the new step direction when the front's marching direction is updated. The direction * only ever RANKS candidates, it never rejects one, so this is a preference and not a limit. ---*/ - constexpr su2double DIR_BLEND = 0.5; + constexpr passivedouble DIR_BLEND = 0.5; - /*--- Safety cap on stack depth, 0 for none. A front is meant to run until it reaches a boundary or - * the mesh stops offering a clean extrusion, so this is off by default. ---*/ + /*--- How many parallel implicit lines may be merged tangential to the wall into one coarse CV. + * 0 selects the dimension-appropriate default: 2 in 2D, 4 in 3D (a wall quad/hex corner). ---*/ unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; From 064adf45bfe23aba682f51fe9c06d5614591c9f5 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 13:12:29 +0200 Subject: [PATCH 46/54] cleanup, remove ILU and PREC_FREEZE stuff --- Common/include/geometry/CGeometry.hpp | 6 - .../include/geometry/CMultiGridGeometry.hpp | 48 +- Common/include/linear_algebra/CSysSolve.hpp | 7 - Common/include/option_structure.hpp | 6 - Common/src/CConfig.cpp | 9 - Common/src/geometry/CGeometry.cpp | 105 +-- Common/src/geometry/CMultiGridGeometry.cpp | 601 ++++++------------ Common/src/linear_algebra/CSysMatrix.cpp | 7 - Common/src/linear_algebra/CSysSolve.cpp | 30 +- 9 files changed, 239 insertions(+), 580 deletions(-) diff --git a/Common/include/geometry/CGeometry.hpp b/Common/include/geometry/CGeometry.hpp index 54d0d76c0ec..9d3e574b432 100644 --- a/Common/include/geometry/CGeometry.hpp +++ b/Common/include/geometry/CGeometry.hpp @@ -228,12 +228,6 @@ class CGeometry { std::vector> linelets; /*!< \brief Point indices for each linelet. */ - /*!< \brief Whether the structure has been built. Not the same as having any linelets: a rank whose - * part of the mesh holds no solid wall builds an empty set, and inferring "not built yet" from - * that emptiness makes it rebuild - and re-run the collectives at the end of the construction - - * on every call, while every other rank answers from its cache and never calls them again. */ - bool built = false; - /*!< \brief Index of the linelet of each point ("linelets" transfered to points). */ std::vector lineletIdx; diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index beaafee5fbe..72a7ddb4592 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -51,12 +51,8 @@ class CMultiGridGeometry final : public CGeometry { const CConfig* config, const vector& mixedBC) const; /*! - * \brief Nodes carrying two or more physical boundary conditions of DIFFERENT type, e.g. the point - * where a wall ends against an outlet. Nishikawa's rules never agglomerate these: merging one - * into a coarse control volume averages two conditions that the fine grid applies separately, - * and neither ends up applied where it belongs. Two markers of the SAME type meeting - two - * wall patches, say - are not affected, nor is a node whose only second marker is - * SEND_RECEIVE, which records a partition and not a boundary condition. + * \brief Find nodes where two boundary conditions of different type meet. These are never + * agglomerated, since a coarse CV holding one would average both conditions. * \param[in] fine_grid - Geometrical definition of the problem. * \param[in] config - Definition of the particular problem. * \return One flag per fine grid point, set where that point must stay on its own. @@ -91,7 +87,7 @@ class CMultiGridGeometry final : public CGeometry { su2double ComputeLocalCurvature(const CGeometry* fine_grid, unsigned long iPoint, unsigned short iMarker) const; /*! - * \brief Agglomerate high-aspect-ratio interior cells along implicit lines from wall vertices. + * \brief Pave the domain with advancing fronts rising from the boundary patches. * \param[in,out] Index_CoarseCV - Current coarse CV index, incremented as new coarse CVs are created. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Configuration. @@ -99,9 +95,8 @@ class CMultiGridGeometry final : public CGeometry { void AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config); /*! - * \brief Per-node dual-grid stiffness data used by the implicit-line agglomeration: the weakest and - * strongest coupling at each node, and the neighbour the strongest one leads to. Their ratio is - * the local cell aspect ratio, available on every multigrid level unlike CGeometry::Aspect_Ratio. + * \brief Weakest and strongest dual-grid coupling at each node, and the neighbour across the + * strongest edge. Their ratio is the local cell aspect ratio, available on every MG level. */ struct CNodeStiffness { vector wMin, wMax; /*!< \brief Weakest and strongest edge coupling at each node. */ @@ -116,33 +111,28 @@ class CMultiGridGeometry final : public CGeometry { /*! * \brief Measure the dual-grid coupling at every node of a grid. * \param[in] fine_grid - Grid to measure. - * \return Weakest/strongest coupling per node, see CNodeStiffness. + * \return Weakest/strongest coupling per node. */ CNodeStiffness ComputeNodeStiffness(const CGeometry* fine_grid) const; /*! - * \brief Diagnostic tally of why each front stopped advancing, indexed by the STOP_* constants - * below. - * - * A front is stopped by exactly two things: reaching a boundary, or failing to lay a next layer - * that is topologically identical to the one it is standing on. Every reason below is one of those - * two. There is deliberately no criterion on direction or on how stretched the mesh is - a front - * runs until the mesh itself stops offering a clean extrusion. + * \brief Why a front stopped advancing. Either it reached a boundary, or it could not lay a layer + * isomorphic to the current one; every reason below is one of those two. */ enum { - STOP_PHYS_BOUNDARY = 0, /*!< \brief Reached a boundary. The one expected, correct stop. */ - STOP_PARTITION, /*!< \brief Reached a partition interface, where the layer cannot be claimed. */ - STOP_COLLISION, /*!< \brief Lost a candidate to another front, i.e. the fronts met. */ + STOP_PHYS_BOUNDARY = 0, /*!< \brief Reached a boundary, the expected stop. */ + STOP_PARTITION, /*!< \brief Reached a partition interface. */ + STOP_COLLISION, /*!< \brief Lost a candidate to another front. */ STOP_PINCH, /*!< \brief Two nodes of this front wanted the same successor. */ - STOP_AGGLOMERATED, /*!< \brief Ran into nodes an earlier phase had already taken. */ - STOP_NO_NEIGHBOR, /*!< \brief A front node had no free neighbour left to step onto. */ + STOP_AGGLOMERATED, /*!< \brief Ran into nodes an earlier phase had taken. */ + STOP_NO_NEIGHBOR, /*!< \brief A front node had no free neighbour left. */ STOP_TOPOLOGY, /*!< \brief The next layer was not isomorphic to the current one. */ STOP_GEOMETRY, /*!< \brief A node of the next layer failed GeometricalCheck. */ N_STOP_REASONS }; /*! - * \brief Boundary nodes that seed an advancing front, with the direction each starts marching in. + * \brief Boundary nodes that seed a front, with the direction each starts marching in. */ struct CFrontSeeds { vector node; /*!< \brief Seed node on the boundary. */ @@ -150,9 +140,8 @@ class CMultiGridGeometry final : public CGeometry { }; /*! - * \brief PHASE 1a of the paving agglomeration: collect the boundary nodes that seed an advancing - * front, i.e. those on a viscous wall, or on another boundary that carries a stretched - * layer normal to itself. + * \brief Collect the boundary nodes that seed an advancing front: those on a viscous wall, or on a + * boundary carrying a stretched layer normal to itself. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Definition of the particular problem. * \param[in] stiff - Node coupling from ComputeNodeStiffness. @@ -161,9 +150,8 @@ class CMultiGridGeometry final : public CGeometry { CFrontSeeds SeedFrontNodes(const CGeometry* fine_grid, const CConfig* config, const CNodeStiffness& stiff) const; /*! - * \brief PHASE 1b of the paving agglomeration: partition the seed nodes into compact surface - * patches by repeated pairwise matching. Each patch is the footprint of one front, and is - * the only thing that decides the shape of the whole stack above it. + * \brief Partition the seed nodes into compact surface patches by repeated pairwise matching. Each + * patch is the footprint of one front and fixes the shape of the stack above it. * \param[in] seeds - Seed nodes from SeedFrontNodes. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Definition of the particular problem. diff --git a/Common/include/linear_algebra/CSysSolve.hpp b/Common/include/linear_algebra/CSysSolve.hpp index 4191b0cf11c..1f9bc851b92 100644 --- a/Common/include/linear_algebra/CSysSolve.hpp +++ b/Common/include/linear_algebra/CSysSolve.hpp @@ -136,13 +136,6 @@ class CSysSolve { /*!< \brief Inner solver for nested preconditioning. */ std::unique_ptr> inner_solver; - /*--- Preconditioner freezing on coarse multigrid levels. The factorization lives in the - * CSysMatrix (not in the short-lived CPreconditioner object built in Solve), so simply - * skipping Build() reuses the previous one. This instance belongs to one solver on one - * grid level, so the counter is naturally per-level. See MG_COARSE_PREC_FREEZE. ---*/ - mutable unsigned long precSolveCount = 0; /*!< \brief Linear solves done by this instance. */ - mutable bool buildPrecThisSolve = true; /*!< \brief Decision for the current solve, shared by all threads. */ - /*! * \brief sign transfer function * \param[in] x - value having sign prescribed diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index d48261ce391..9440ecc5e0b 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1128,12 +1128,6 @@ struct CMGOptions { unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse CV tangential to the wall. 0 = dimension-appropriate default (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ - su2double MG_Implicit_Lines_Min_AR{2.0}; /*!< \brief Smallest local cell aspect ratio for which a node still counts - as part of a stretched layer. Decides which non-wall boundaries carry - a layer normal to them and may therefore seed paving fronts. It is a - SEEDING gate only and never stops a front that has started. - See CMultiGridGeometry::SeedFrontNodes. */ - unsigned long MG_Coarse_Prec_Freeze{1}; /*!< \brief On MG levels > 0, reuse the linear-solver preconditioner for this many consecutive solves. 1 = rebuild every solve. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ su2double MG_Startup_Convergence{-2.0}; /*!< \brief FMG: orders of magnitude (log10) that CONV_FIELD must drop on the active level before promoting to the next finer one. Negative is a diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index 4cc52a2ff79..ee2036c598f 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2065,10 +2065,6 @@ void CConfig::SetConfig_Options() { addDoubleOption("MG_SMOOTH_STAGNATION_TOL", MGOptions.MG_Smooth_StagnationTol, 0.99); /*!\brief MG_SMOOTH_COEFF\n DESCRIPTION: Smoothing coefficient for the correction prolongation Jacobi smoother. DEFAULT: 1.25 \ingroup Config*/ addDoubleOption("MG_SMOOTH_COEFF", MGOptions.MG_Smooth_Coeff, 1.25); - /*!\brief MG_COARSE_PREC_FREEZE\n DESCRIPTION: On multigrid levels above MESH_0, reuse the linear-solver preconditioner - * (e.g. the ILU factorization) for this many consecutive linear solves instead of rebuilding it every time. - * 1 reproduces the previous behaviour exactly. DEFAULT: 1 \ingroup Config*/ - addUnsignedLongOption("MG_COARSE_PREC_FREEZE", MGOptions.MG_Coarse_Prec_Freeze, 1); /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 50 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ @@ -2077,11 +2073,6 @@ void CConfig::SetConfig_Options() { * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_GROUP", MGOptions.MG_Implicit_Lines_Max_Group, 0); - /*!\brief MG_IMPLICIT_LINES_MIN_AR\n DESCRIPTION: Smallest local cell aspect ratio for which a node still counts as - * part of a stretched layer, measured from the ratio of dual-grid edge weights. Decides which non-wall boundaries - * carry a layer normal to them and may therefore seed paving fronts; viscous walls always seed. This is a seeding - * gate only and never stops a front that has started. 1.0 lets every boundary seed. DEFAULT: 2.0 \ingroup Config*/ - addDoubleOption("MG_IMPLICIT_LINES_MIN_AR", MGOptions.MG_Implicit_Lines_Min_AR, 2.0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); diff --git a/Common/src/geometry/CGeometry.cpp b/Common/src/geometry/CGeometry.cpp index dab1417895e..360da1aaaa1 100644 --- a/Common/src/geometry/CGeometry.cpp +++ b/Common/src/geometry/CGeometry.cpp @@ -25,7 +25,6 @@ * License along with SU2. If not, see . */ -#include #include #include "../../include/geometry/CGeometry.hpp" @@ -4340,8 +4339,7 @@ void CGeometry::ColorMGLevels(unsigned short nMGLevels, const CGeometry* const* const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) const { auto& li = lineletInfo; - if (li.built || nPoint == 0) return li; - li.built = true; + if (!li.linelets.empty() || nPoint == 0) return li; li.lineletIdx.resize(nPoint, CLineletInfo::NO_LINELET); @@ -4358,19 +4356,6 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) unsigned long maxNPoints = 0, sumNPoints = 0; - /*--- Why each line stopped growing. A line that ends because the mesh has become isotropic is a - * line that has done its job; one that ends because no neighbour was well enough aligned means - * the walk lost the wall-normal direction and the line is short despite the mesh still being - * stretched. The two call for opposite responses on coarse grids, so they are counted apart. ---*/ - unsigned long nStopIsotropic = 0, nStopNoNeighbour = 0, nStopCap = 0; - /*--- "No neighbour" has two very different causes: every candidate was already taken by another - * line (a competition/ordering problem), or candidates were free but none lay within 45 deg of - * the current direction (a geometry problem). Only the second says the mesh lost its - * wall-normal structure. For the latter, also accumulate the best alignment on offer, which - * says whether the 45 deg threshold is merely too tight or the direction is truly lost. ---*/ - unsigned long nStopAllTaken = 0, nStopMisaligned = 0; - su2double sumBestCos = 0.0; - if (nLinelet != 0) { /*--- Define the basic linelets, starting from each vertex, preventing duplication of points. ---*/ @@ -4390,23 +4375,11 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } li.linelets.resize(nLinelet); - /*--- Grow the lines breadth first: each pass advances every still-growing line by one point. - * Growing them depth first - running one line to completion before starting the next - lets an - * early line exhaust its own column, turn sideways (the 45 deg test permits it) and consume the - * points its neighbours needed, starving them into one- and two-point stubs. That is harmless on - * the fine grid, where the boundary layer is deeper than MAX_LINELET_POINTS so no line ever runs - * out of vertical room, but on agglomerated grids the layer is shallower than the cap and the - * starvation is severe. Advancing in lockstep makes the lines compete on equal terms for the - * layer they are all entitled to. ---*/ - - std::vector growing(nLinelet, 1); + /*--- Create the linelet structure. ---*/ - for (unsigned long step = 1; step < CLineletInfo::MAX_LINELET_POINTS; ++step) { - bool anyGrew = false; - - for (auto iLine = 0ul; iLine < nLinelet; ++iLine) { - if (!growing[iLine]) continue; - auto& linelet = li.linelets[iLine]; + nLinelet = 0; + for (auto& linelet : li.linelets) { + while (linelet.size() < CLineletInfo::MAX_LINELET_POINTS) { const auto iPoint = linelet.back(); /*--- Compute the value of the max and min weights to detect if this region is isotropic. ---*/ @@ -4425,35 +4398,26 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } /*--- Isotropic, stop this linelet. ---*/ - if (min_weight / max_weight > CLineletInfo::ALPHA_ISOTROPIC()) { - growing[iLine] = 0; - ++nStopIsotropic; - continue; - } + if (min_weight / max_weight > CLineletInfo::ALPHA_ISOTROPIC()) break; /*--- Otherwise, add the closest valid neighbor. ---*/ su2double min_dist2 = std::numeric_limits::max(); auto next_Point = iPoint; const auto* iCoord = nodes->GetCoord(iPoint); - unsigned long nFreeCandidates = 0; - su2double bestCos = -1.0; for (const auto jPoint : nodes->GetPoints(iPoint)) { if (li.lineletIdx[jPoint] == CLineletInfo::NO_LINELET && nodes->GetDomain(jPoint)) { - ++nFreeCandidates; const auto* jCoord = nodes->GetCoord(jPoint); const su2double d2 = GeometryToolbox::SquaredDistance(nDim, iCoord, jCoord); su2double cosTheta = 1; - su2double dij[3] = {0.0}; - GeometryToolbox::Distance(nDim, jCoord, iCoord, dij); if (linelet.size() > 1) { const auto* kCoord = nodes->GetCoord(linelet[linelet.size() - 2]); - su2double dki[3] = {0.0}; + su2double dij[3] = {0.0}, dki[3] = {0.0}; GeometryToolbox::Distance(nDim, iCoord, kCoord, dki); + GeometryToolbox::Distance(nDim, jCoord, iCoord, dij); cosTheta = GeometryToolbox::DotProduct(3, dki, dij) / sqrt(d2 * GeometryToolbox::SquaredNorm(nDim, dki)); } - bestCos = max(bestCos, cosTheta); if (d2 < min_dist2 && cosTheta > 0.7071) { next_Point = jPoint; min_dist2 = d2; @@ -4462,61 +4426,28 @@ const CGeometry::CLineletInfo& CGeometry::GetLineletInfo(const CConfig* config) } /*--- Did not find a suitable point. ---*/ - if (next_Point == iPoint) { - growing[iLine] = 0; - ++nStopNoNeighbour; - if (nFreeCandidates == 0) { - ++nStopAllTaken; - } else { - ++nStopMisaligned; - sumBestCos += bestCos; - } - continue; - } + if (next_Point == iPoint) break; linelet.push_back(next_Point); - li.lineletIdx[next_Point] = iLine; - anyGrew = true; + li.lineletIdx[next_Point] = nLinelet; } + ++nLinelet; - if (!anyGrew) break; - } - - /*--- A line that never stopped advancing ran into the length cap. ---*/ - for (auto iLine = 0ul; iLine < nLinelet; ++iLine) { - if (growing[iLine]) ++nStopCap; - maxNPoints = max(maxNPoints, li.linelets[iLine].size()); - sumNPoints += li.linelets[iLine].size(); + maxNPoints = max(maxNPoints, linelet.size()); + sumNPoints += linelet.size(); } } /*--- Average linelet size over all ranks. ---*/ - unsigned long globalNPoints, globalNLineLets, globalMaxNPoints; - unsigned long stopCounts[5] = {nStopIsotropic, nStopNoNeighbour, nStopCap, nStopAllTaken, nStopMisaligned}; - unsigned long globalStop[5] = {}; + unsigned long globalNPoints, globalNLineLets; SU2_MPI::Allreduce(&sumNPoints, &globalNPoints, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&nLinelet, &globalNLineLets, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - SU2_MPI::Allreduce(&maxNPoints, &globalMaxNPoints, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); - SU2_MPI::Allreduce(stopCounts, globalStop, 5, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - su2double globalSumBestCos = 0.0; - SU2_MPI::Allreduce(&sumBestCos, &globalSumBestCos, 1, MPI_DOUBLE, MPI_SUM, SU2_MPI::GetComm()); - - if (rank == MASTER_NODE && globalNLineLets > 0) { - const auto pct = [&](unsigned long n) { return 100.0 * passivedouble(n) / globalNLineLets; }; - std::cout << "Computed linelet structure on MG level " << MGLevel << ", " + + if (rank == MASTER_NODE) { + std::cout << "Computed linelet structure, " << static_cast(passivedouble(globalNPoints) / globalNLineLets) - << " points in each line (average), " << globalMaxNPoints << " longest, " << globalNLineLets - << " lines.\n" - << " Line ends because: " << pct(globalStop[0]) << "% mesh became isotropic, " << pct(globalStop[1]) - << "% no aligned neighbour, " << pct(globalStop[2]) << "% hit the " << CLineletInfo::MAX_LINELET_POINTS - << "-point cap.\n" - << " of the 'no aligned neighbour': " << pct(globalStop[3]) - << "% all candidates already claimed by another line, " << pct(globalStop[4]) - << "% candidates free but misaligned"; - if (globalStop[4] > 0) - std::cout << " (best cos on offer " << globalSumBestCos / globalStop[4] << ", need > 0.7071)"; - std::cout << "." << std::endl; + << " points in each line (average)." << std::endl; } /*--- Color the linelets for OpenMP parallelization and visualization. ---*/ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 948284682b0..3f3889c43b5 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -89,14 +89,9 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } /*--- STEP 0: pave the domain with advancing fronts rising from the boundaries, wall CV included. - * This runs before the general boundary agglomeration below so that the wall control volume and - * the layers stacked on top of it share one footprint; letting the general scheme claim the wall - * first would fix a footprint chosen without any knowledge of the fronts, and the stack above it - * could then only be misaligned with its own base. Everything it claims is - * already marked agglomerated, so the boundary and interior passes below simply skip it. - * - * The coarse CVs it creates occupy the half-open index range [firstLineCV, endLineCV), which is - * how the repair passes further down tell them apart from the rest, see isStackBase. ---*/ + * It runs before the boundary agglomeration so that the wall CV and the layers above it share one + * footprint. Everything it claims is marked agglomerated, so the passes below skip it. The coarse + * CVs it creates occupy [firstLineCV, endLineCV), which is how the repair passes identify them. ---*/ const auto firstLineCV = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config); @@ -123,11 +118,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- Skip periodic boundaries: do not agglomerate on periodic markers. ---*/ if (config->GetMarker_All_KindBC(iMarker) == PERIODIC_BOUNDARY) continue; - /*--- Skip SEND_RECEIVE markers. Carrying one does not put a point on a boundary, it only - * records that the point is mirrored on another rank. A point whose only markers are - * SEND_RECEIVE is an interior point, and is left to the domain pass (STEP 2) which is - * where a serial run would agglomerate it too. Points that do sit on a physical boundary - * are still reached here through their physical marker. ---*/ + /*--- Skip SEND_RECEIVE markers: they record that a point is mirrored on another rank, not that it + * lies on a boundary. Such points are left to the domain pass, as in a serial run. ---*/ if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { @@ -280,11 +272,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Only take into account indirect neighbors for 3D faces, not 2D. The size test has to be - * made on entry as well: the sweep above leaves with the CV exactly full whenever it hit - * the limit, and without a guard here the first indirect candidate pushed it to nine - * children, after which the equality test below could never match again and the CV grew - * without a bound at all. ---*/ + /*--- Indirect neighbors only for 3D faces. The size test is needed on entry too: the sweep + * above leaves the CV exactly full when it hit the limit, and without this the first + * indirect candidate pushed it past the limit, after which the equality test never matched + * and the CV grew unbounded. ---*/ if ((nDim == 3) && (nChildren < maxAgglomSize)) { Suitable_Indirect_Neighbors.clear(); @@ -365,45 +356,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- STEP 2: Agglomerate the domain points. - * - * A seed is grown one node at a time, and at every step the node that joins is the one most tied - * to what the CV already holds, i.e. the candidate sharing the most edges with its current members. - * Taking the seed's neighbours in whatever order the connectivity happens to list them, as this - * used to, ignores the shape of the result: on a hex mesh the seed has one neighbour per axis, so - * the first few are picked from different axes and the CV grows into a star. The leftovers around - * it then have to be swept up by later seeds, which is where the ragged pieces come from - a CV - * holding three nodes in one mesh plane and a single node in the next reads, in a cross-section - * through the second plane, as an isolated node sitting in the corner of an L. - * - * Counting shared edges instead closes those shapes off by construction. Once a seed has taken two - * nodes along different axes, the node diagonally between them touches two members while every - * other option still touches one, so it wins and completes the square; the same argument then - * repeats one axis up and completes the cube. On a structured hex mesh the result is an exact - * 2x2x2 block, which is what the implicit-line stacks already produce and what the isotropic - * region should match. - * - * Ties are settled by distance to the centroid, but that distance has to be measured in cells and - * not in metres. Until the CV holds an L there is nothing for the shared count to prefer - on a hex - * graph the node diagonal to two members is not adjacent to the seed, so every candidate shares - * exactly one face and the distance decides alone. In a stretched cell the neighbour across the - * thin direction is nearer than any neighbour along the layer by whatever the aspect ratio happens - * to be, so the CV steps that way, and then finds the next step in the SAME direction nearer still. - * It walks the boundary layer end to end: measured on a real mesh, 63% of the CVs built here came - * out as eight nodes in a straight line. That is the wrong shape, and worse, the wrong direction to - * coarsen in, since it merges exactly the wall-normal cells the implicit lines exist to keep apart. - * - * The yardstick is the seed's own incident edges: a candidate's offset is divided by the length of - * the edge pointing most nearly the same way. A step across the layer and a step along it then both - * come to about one, whatever the stretching, and the shared count takes over from there. Choosing - * the edge by direction rather than by which Cartesian axis it is closest to is what keeps this - * usable on a curved boundary, where the wall-normal direction is not any one axis and a per-axis - * spacing would be measuring the wrong thing over most of the surface. - * - * The candidate set grows with the CV rather than being fixed to the seed's own neighbours: the - * far corner of a cube is not adjacent to the seed, it only becomes reachable once the nodes - * between them have joined. That also makes SetSuitableNeighbors unnecessary here, since the nodes - * it used to supply are reached through ordinary edges as the frontier advances. ---*/ + /*--- STEP 2: Agglomerate the domain points. A seed grows one node at a time, taking the candidate + * that shares the most edges with the current members so the CV closes into a block rather than a + * star. Ties break on distance to the centroid measured in cells, using a local frame built from + * the seed's own edges, so a stretched cell does not pull the CV along the boundary layer. ---*/ /*--- Scratch shared by all seeds. The markers are cleared per CV, touching only what was used. ---*/ vector inCV(fine_grid->GetnPoint(), 0); @@ -411,12 +367,9 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un vector members, candidates; members.reserve(maxAgglomSize); - /*--- A local frame at the current seed, up to nDim of its incident edges chosen to be as mutually - * orthogonal as possible, each with its own length. It is the yardstick described above: an - * offset is resolved onto these directions and each component divided by that direction's own - * spacing. Being built from the edges themselves it turns with the mesh, so it still measures - * the wall-normal direction correctly where a boundary curves away from any Cartesian axis, and - * on an axis-aligned mesh it reduces to dividing x, y and z by their own spacings. ---*/ + /*--- A local frame at the seed: up to nDim of its incident edges, as mutually orthogonal as possible, + * each with its own length. An offset is resolved onto these and each component divided by that + * direction's spacing, so the measure turns with the mesh and stays correct on a curved boundary. ---*/ vector> frameDir, edgeDir; vector frameLen, edgeLen; vector edgeUsed; @@ -603,19 +556,13 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, but only for the CVs that really do border another rank rather than for every CV near one. ---*/ - /*--- Coarse CVs that must be left exactly as the agglomeration made them: those holding a node - * where two different boundary conditions meet. Both repair passes below exist to get rid of - * one-child control volumes, and a deliberately isolated junction IS a one-child control volume, - * so without this they simply undo the isolation. They have to be protected as a TARGET as well - * as a source: pass two merges a singleton into its smallest neighbour, and a one-child junction - * CV is by construction the smallest neighbour there is. ---*/ + /*--- Coarse CVs to leave exactly as the agglomeration made them: those holding a node where two + * different boundary conditions meet. Both repair passes remove one-child CVs, and a deliberately + * isolated junction is one, so they must be protected as a merge TARGET as well as a source. ---*/ vector mustStayAlone(nPointDomain, false); - /*--- ...and which coarse CVs hold a boundary node at all. A boundary node is never agglomerated - * with an interior one, which the agglomeration itself already respects - the first advance of - * every front is a single layer, so the boundary layer is its own CV. The repair passes below - * would otherwise undo it from the other end: they merge a one-child CV into a neighbour, and a - * boundary CV's neighbours include the interior CV sitting on top of it. On the flat plate they - * happened to pick a boundary neighbour every time, which is luck rather than a rule. ---*/ + /*--- ...and which coarse CVs hold a boundary node at all. A boundary node is never agglomerated with + * an interior one, which the paving already respects. The repair passes below would undo it from + * the other end, since a boundary CV's neighbours include the interior CV sitting on top of it. ---*/ vector cvOnBoundary(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -624,14 +571,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (onPhysBoundary[iFinePoint]) cvOnBoundary[iCoarsePoint] = true; } - /*--- Which physical boundaries each coarse CV sits on, one bit per marker. cvOnBoundary above only - * records THAT a CV touches a boundary, and the repair passes below compare nothing else, so a - * one-child CV on boundary A is free to be merged into a neighbour that lies on boundary B. That - * hands the target CV a marker none of its own children carried: SetVertex gives a coarse CV - * every marker of every child, so the merged CV becomes a vertex of A while its centroid sits - * wherever the B stack put it, and the boundary condition for A is then applied to a control - * volume that is mostly not on A. Comparing the marker SETS - a strictly stronger test than the - * boolean, since an interior CV has an empty set - keeps a merge inside one boundary. ---*/ + /*--- Which physical boundaries each coarse CV sits on, one bit per marker. cvOnBoundary records only + * THAT a CV touches a boundary, so a one-child CV on boundary A could be merged into a neighbour on + * boundary B; SetVertex would then give the merged CV marker A while its body sits on B. Comparing + * marker sets is strictly stronger and keeps a merge inside one boundary. ---*/ vector cvMarkerMask(nPointDomain, 0); { vector bitOfMarker(fine_grid->GetnMarker(), -1); @@ -658,17 +601,10 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- A boundary CV built by the paving is the BASE of a stack, and the paving's whole guarantee is - * that every layer above it was given that same footprint. The repair passes must not take the - * footprint away: merging a stack base into the neighbouring stack leaves the column above it - * headless and the merged base wider than either column, so base and first layer no longer line - * up. On the next level that misalignment is fatal rather than cosmetic - the stiffest neighbour - * of the widened base is then a LATERAL one, SeedFrontNodes' hasLayerNormalTo rejects it, the CV - * does not seed a front, and it is swallowed mid-stack by an interior front instead. The coarse - * CV that results carries the boundary marker with its body off the boundary, and the boundary - * condition is applied to it. A one-node patch marching as a one-wide stack is a deliberate - * choice in AgglomerateImplicitLines, not damage for these passes to repair; what they are for - * is the INTERIOR singleton left where a line narrows, and that is untouched by this. ---*/ + /*--- A boundary CV built by the paving is the base of a stack and must keep its footprint. Merging + * one into a neighbour leaves the column headless and widens the base, which on the next level + * stops it seeding a front and lets an interior front swallow it mid-stack. The repair passes + * below are for the interior singleton left where a line narrows, not for these. ---*/ auto isStackBase = [&](unsigned long iCoarsePoint) { return cvOnBoundary[iCoarsePoint] && (iCoarsePoint >= firstLineCV) && (iCoarsePoint < endLineCV); }; @@ -1684,15 +1620,10 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, } CMultiGridGeometry::CNodeStiffness CMultiGridGeometry::ComputeNodeStiffness(const CGeometry* fine_grid) const { - /*--- Strength of the coupling across the dual face between a node and one of its neighbours. For a cell - * of streamwise size dx and wall-normal size dy this is 1/dy across the wall-normal face and 1/dx - * across the tangential one, so the ratio of the largest weight at a node to the smallest is the - * local cell aspect ratio. That makes the aspect ratio available from the dual grid alone, which - * SetControlVolume builds on every multigrid level, whereas CGeometry::Aspect_Ratio exists only on - * MESH_0. The same quantity decides where LINELET preconditioner lines stop, in GetLineletInfo. - * - * Measuring the whole grid once keeps this out of the line-growth loop, which previously rescanned - * every neighbour of a node to find its weakest edge on every step of every line. ---*/ + /*--- Coupling across the dual face between a node and a neighbour. For a cell of size dx by dy this + * is 1/dy across the wall-normal face and 1/dx across the tangential one, so the ratio of largest + * to smallest weight at a node is the local aspect ratio. Unlike CGeometry::Aspect_Ratio this is + * available on every multigrid level, since SetControlVolume builds the dual grid on each. ---*/ const auto nPointFine = fine_grid->GetnPoint(); CNodeStiffness stiff; @@ -1749,11 +1680,13 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet /*--- Fraction of a marker's nodes that must sit in a layer before the whole marker may seed. ---*/ constexpr passivedouble QUALIFIED_FRACTION = 0.5; constexpr passivedouble ANGLE_THRESHOLD_DEG = 30.0; + /*--- Smallest local cell aspect ratio for which a node still counts as part of a stretched layer. + * Not exposed as an option: the two populations it separates are decades apart - a boundary + * carrying a layer normal to itself measures in the hundreds or more, one in isotropic mesh + * measures about one - so every value between them selects the same set of markers. ---*/ + constexpr passivedouble MIN_AR = 2.0; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); - const su2double MIN_AR = config->GetMGOptions().MG_Implicit_Lines_Min_AR; - const bool USE_AR = (MIN_AR > 1.0); - const auto nMarkerFine = fine_grid->GetnMarker(); constexpr auto NO_POINT = std::numeric_limits::max(); @@ -1804,65 +1737,55 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) if (isWall(config->GetMarker_All_KindBC(iMarker))) seedMarker(iMarker, false); - /*--- Boundaries other than viscous walls that nevertheless carry a stretched layer normal to - * themselves, such as a symmetry plane laid in the same surface as a wall - the floor either side - * of a bump, or ahead of a flat plate's leading edge. Seeding only from walls leaves the mesh - * above those to isotropic agglomeration, so the coarse grid changes character across a line the - * fine grid does not single out. - * - * The verdict is taken per marker, not per node: seeding isolated qualifying nodes on a marker - * that mostly does not qualify scatters one-node patches, i.e. coarse CVs that do not coarsen - * tangentially at all. The two populations are far apart in practice - boundaries with a layer - * normal to them qualify at 100%, side planes, inlets and far fields at 13% and below - so any - * threshold near a half separates them. ---*/ - if (USE_AR) { - /*--- Counted per configuration-file marker, not per local marker: ranks agree on neither the - * number nor the order of local markers, because each partition appends its own SEND_RECEIVE - * markers, so the same index means a different boundary elsewhere. ---*/ - const auto nMarkerCfg = config->GetnMarker_CfgFile(); - vector nValid(nMarkerCfg, 0), nQualified(nMarkerCfg, 0); - - auto canSeed = [&](unsigned short iMarker) { - const auto bc = config->GetMarker_All_KindBC(iMarker); - /*--- Periodic boundaries are left out: the two halves are the same physical location under a - * transform and have their own matching, which a front running into one would disturb. ---*/ - return (bc != SEND_RECEIVE) && (bc != PERIODIC_BOUNDARY) && !isWall(bc); - }; + /*--- Non-wall boundaries that still carry a layer normal to themselves, such as a symmetry plane in + * the same surface as a wall. The verdict is per marker, not per node: seeding isolated qualifying + * nodes scatters one-node patches that do not coarsen tangentially at all. ---*/ + /*--- Counted per configuration-file marker, not per local marker: ranks agree on neither the + * number nor the order of local markers, because each partition appends its own SEND_RECEIVE + * markers, so the same index means a different boundary elsewhere. ---*/ + const auto nMarkerCfg = config->GetnMarker_CfgFile(); + vector nValid(nMarkerCfg, 0), nQualified(nMarkerCfg, 0); + + auto canSeed = [&](unsigned short iMarker) { + const auto bc = config->GetMarker_All_KindBC(iMarker); + /*--- Periodic boundaries are left out: the two halves are the same physical location under a + * transform and have their own matching, which a front running into one would disturb. ---*/ + return (bc != SEND_RECEIVE) && (bc != PERIODIC_BOUNDARY) && !isWall(bc); + }; - vector cfgOfMarker(nMarkerFine, 0); - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) - if (canSeed(iMarker)) - cfgOfMarker[iMarker] = config->GetMarker_CfgFile_TagBound(config->GetMarker_All_TagBound(iMarker)); + vector cfgOfMarker(nMarkerFine, 0); + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) + if (canSeed(iMarker)) + cfgOfMarker[iMarker] = config->GetMarker_CfgFile_TagBound(config->GetMarker_All_TagBound(iMarker)); - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { - if (!canSeed(iMarker)) continue; - for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { - const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); - if (!fine_grid->nodes->GetDomain(iPoint)) continue; - su2double Normal[MAXNDIM] = {0.0}; - if (!VertexUnitNormal(fine_grid, nDim, iPoint, iMarker, Normal)) continue; - nValid[cfgOfMarker[iMarker]]++; - if (hasLayerNormalTo(iPoint, Normal)) nQualified[cfgOfMarker[iMarker]]++; - } + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (!canSeed(iMarker)) continue; + for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { + const auto iPoint = fine_grid->vertex[iMarker][iVertex]->GetNode(); + if (!fine_grid->nodes->GetDomain(iPoint)) continue; + su2double Normal[MAXNDIM] = {0.0}; + if (!VertexUnitNormal(fine_grid, nDim, iPoint, iMarker, Normal)) continue; + nValid[cfgOfMarker[iMarker]]++; + if (hasLayerNormalTo(iPoint, Normal)) nQualified[cfgOfMarker[iMarker]]++; } + } - /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. - * Every rank reaches these collectives, including one that owns no boundary at all. ---*/ - if (nMarkerCfg > 0) { - vector tmp(nMarkerCfg); - SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - nValid.swap(tmp); - SU2_MPI::Allreduce(nQualified.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - nQualified.swap(tmp); - } + /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. + * Every rank reaches these collectives, including one that owns no boundary at all. ---*/ + if (nMarkerCfg > 0) { + vector tmp(nMarkerCfg); + SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + nValid.swap(tmp); + SU2_MPI::Allreduce(nQualified.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + nQualified.swap(tmp); + } - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { - if (!canSeed(iMarker)) continue; - const auto iCfg = cfgOfMarker[iMarker]; - if (nValid[iCfg] == 0) continue; - if (su2double(nQualified[iCfg]) < QUALIFIED_FRACTION * su2double(nValid[iCfg])) continue; - seedMarker(iMarker, true); - } + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { + if (!canSeed(iMarker)) continue; + const auto iCfg = cfgOfMarker[iMarker]; + if (nValid[iCfg] == 0) continue; + if (su2double(nQualified[iCfg]) < QUALIFIED_FRACTION * su2double(nValid[iCfg])) continue; + seedMarker(iMarker, true); } return seeds; @@ -1871,23 +1794,10 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet vector> CMultiGridGeometry::BuildFrontPatches(const CFrontSeeds& seeds, const CGeometry* fine_grid, const CConfig* config, const vector& mixedBC) const { - /*--- Every seed must end up in exactly one patch, and a patch must be compact: in 3D the four nodes - * of one boundary quadrilateral, in 2D the two ends of one boundary edge. Choosing, for each seed - * independently, a set of neighbours to merge with does not do this - the relation is not - * symmetric, so seed 1 claiming {2,3} does not stop seed 2 claiming {1,4}, and the patches - * overlap and fight over nodes. - * - * Repeated pairwise matching avoids that by construction. One round pairs adjacent seeds into the - * boundary edge, a second pairs adjacent pairs into the boundary quadrilateral. Each round is a - * matching, so membership stays mutually exclusive and the result is a true partition. It needs - * nothing but point-to-point connectivity, so it works identically on every multigrid level - - * boundary face connectivity does not survive agglomeration, so a literal "same quadrilateral" - * test would only ever work for the first coarsening. Two rounds reach 4, which is the 3D patch - * size, so the number of rounds follows from max_group instead of iterating to a fixed point. - * - * This patch is the ONLY thing that decides the footprint of the stack above it. The front that - * rises from it keeps exactly these nodes' successors, layer after layer, so getting the patch - * square is what makes the coarse CVs square all the way up. ---*/ + /*--- Repeated pairwise matching, one round per doubling, partitions the seeds into compact patches: + * a boundary edge in 2D, a boundary quadrilateral in 3D. Matching keeps membership mutually + * exclusive, which choosing neighbours per seed would not. It needs only point connectivity, so it + * works on every multigrid level. This patch alone fixes the footprint of the stack above it. ---*/ const auto nSeeds = seeds.node.size(); unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; @@ -1998,12 +1908,9 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront for (const auto& t : shared) merges.push_back({g, t.first, t.second, gkey[g], gkey[t.first]}); } - /*--- Best first, over ALL groups at once. Sweeping the groups in index order instead and letting - * each take its own best partner is what produced the strips: a group with no square partner - * left would take a weight-1 merge and consume a group that a later one needed for its square, - * and the failures cascade - on a structured 3D wall that came to 19% of the footprints. - * Ordering the merges globally means every square in the mesh is made before the first strip is - * even considered, so a strip only forms where the surface genuinely offers nothing better. ---*/ + /*--- Best merges first over all groups at once. Sweeping groups in index order lets one with no + * square partner take a weight-1 merge and consume a group a later one needed, and the failures + * cascade. Ordering globally makes every square before the first strip is considered. ---*/ std::sort(merges.begin(), merges.end(), [](const CMerge& a, const CMerge& b) { if (a.weight != b.weight) return a.weight > b.weight; if (a.keyG != b.keyG) return a.keyG < b.keyG; @@ -2035,57 +1942,12 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config) { - /*================================================================================================== - * Paving by advancing fronts. - * - * The boundary is agglomerated first into patches (PHASE 1), and every patch then rises into the - * domain as a FRONT that keeps its footprint: at each round every front node picks a successor, - * and the front advances only if all of them succeeded and none was lost to another front. So the - * layers of one stack are congruent by construction, and a coarse CV can never contain a node that - * belongs over a neighbouring patch. - * - * This is the part the earlier line-based version could not guarantee, and the reason it is gone. - * There, each wall node grew its own line first and the lines were grouped into bundles only - * afterwards, so nothing tied a line to the footprint it would later be asked to share: the - * marching direction was re-set to the last step taken every step, which let a line random-walk - * tangentially one legal 30-degree step at a time into a neighbouring column, and the connectivity - * test that should have caught it was applied to the lines' WALL roots, which stay adjacent no - * matter how far apart their tops drift. Ragged line lengths then made it worse, because a bundle - * was allowed to carry on with whichever subset of its lines was still long enough, so the stack - * changed footprint as it rose. - * - * Three rules replace all of that: - * - * - The front is the primitive. Nothing marches except a whole patch, so there is no such thing as - * an individual line to drift. - * - All-or-nothing layers. A front that cannot fill an entire layer retires and leaves the rest to - * ordinary agglomeration, instead of continuing narrower. - * - Contention resolved globally, once per layer, from bids collected before any is granted. Two - * fronts reaching for the same node is exactly the event "the fronts have met", and it is caught - * in the layer where it happens rather than fifteen layers later. - * - * A front is stopped by TWO things and nothing else: reaching a boundary, or being unable to lay a - * layer topologically identical to the one it is standing on (see layerIsIsomorphic below). There - * is no limit on how far it may turn and no threshold on how stretched the mesh has to be. Those - * limits used to exist and they were the wrong instrument: an aspect-ratio cut in particular stops - * each front on a contour of the LOCAL cell shape, which on a flat plate is a contour of the - * streamwise spacing, so fronts died at different heights and the paved region ended in a - * staircase - and, being driven by dx rather than by the boundary layer, a staircase running the - * wrong way. Without it the same case paves to the far boundary at a uniform depth and the domain - * pass has nothing left to do. - * - * Each coarse CV is the front's footprint taken TWO layers deep, so the paving coarsens by the same - * factor in every direction: a 2x2 wall patch and the two layers above it make the 2x2x2 block, and - * the next CV of the stack starts from the footprint the front already has. Taking one layer per CV - * instead leaves the wall-normal direction uncoarsened entirely, which is what a line-implicit - * smoother wants and not what this is for. The single-layer CV survives in one place only: the top - * of a stack that retires with one layer buffered, where the alternative is dropping it back to - * ordinary agglomeration. - * - * The multigrid queue is deliberately not touched here: the sync loop after the boundary - * agglomeration removes every point already marked agglomerated, so doing it here too would be an - * error. - *================================================================================================*/ + /*--- Paving by advancing fronts. Each boundary patch from PHASE 1 rises into the domain keeping its + * footprint: every front node picks a successor and the front advances only if all succeed, so the + * layers of a stack are congruent and a coarse CV never spans two patches. A front stops at a + * boundary or when the next layer is not isomorphic to the current one. Each coarse CV is the + * footprint two layers deep. The multigrid queue is not touched here; the sync loop after boundary + * agglomeration already removes every point marked agglomerated. ---*/ const auto starting_Index_CoarseCV = Index_CoarseCV; const auto nPointFine = fine_grid->GetnPoint(); const auto nMarkerFine = fine_grid->GetnMarker(); @@ -2114,12 +1976,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); const auto patches = BuildFrontPatches(seeds, fine_grid, config, mixedBC); - /*--- Nodes on a boundary that carries a boundary condition. A front must not grow into one: those - * nodes belong to the boundary agglomeration and a stack absorbing one would straddle two - * boundaries. CPoint's Boundary flag cannot answer this, as it is also set by SEND_RECEIVE, so on - * a partitioned mesh it is true for ordinary interior nodes of the send fringe and every front - * would stop one layer short of the partition. Walking the markers' own vertex lists is both - * exact and cheaper than testing every point against every marker. ---*/ + /*--- Nodes on a boundary carrying a boundary condition; a front must not grow into one or the stack + * would straddle two boundaries. CPoint's Boundary flag is also set by SEND_RECEIVE, so it would + * stop every front one layer short of a partition. ---*/ vector onPhysicalBoundary(nPointFine, 0); for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; @@ -2127,13 +1986,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, onPhysicalBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } - /*--- A node on a physical boundary only ENDS a front if the front is stepping INTO that boundary, - * i.e. the step direction is roughly parallel to the boundary's own normal. A node that merely - * runs ALONG a boundary - a column on a spanwise symmetry plane, say - has that boundary's normal - * roughly PERPENDICULAR to the step, and is a legitimate interior node of the stack, not its end: - * onPhysicalBoundary alone cannot tell these apart, since it only records marker membership, not - * which direction the marker's surface runs in. Without this a front that happens to sit on a - * tangential boundary the whole way up dies at its very first step. ---*/ + /*--- A boundary node ends a front only if the step runs INTO that boundary, i.e. roughly parallel to + * its normal. A node running ALONG a boundary, such as a column on a spanwise symmetry plane, is a + * legitimate interior node of the stack; onPhysicalBoundary records marker membership only and + * cannot tell the two apart. ---*/ auto entersBoundary = [&](unsigned long jPoint, const su2double* stepDir) { for (unsigned short iMarker = 0; iMarker < nMarkerFine; iMarker++) { if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; @@ -2193,7 +2049,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, vector failed; vector failReason; - vector stopCounts(N_STOP_REASONS, 0); /*--- Set when a front hands only PART of its footprint over and goes on marching here with what is * left of it, so the retirement pass at the end of the round knows not to kill it. ---*/ @@ -2225,9 +2080,25 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- Scratch for the layer under construction, hoisted so a front does not allocate per layer. ---*/ vector newLayer; - unsigned long nStacks = 0, nSemiCV = 0, nFullCV = 0, nLayers = 0, nCovered = 0; - unsigned long nHandedOut = 0, nHandedIn = 0; - unsigned long nSplit = 0, nSplitLocal = 0, nSplitHanded = 0, nSplitDropped = 0, nNoHalo = 0; + /*--- Paving diagnostics, summed over all ranks in one reduction at the end. ---*/ + enum { + P_STACKS, + P_LAYERS, + P_COVERED, + P_SEMICV, + P_FULLCV, + P_HANDOUT, + P_HANDIN, + P_SPLIT, + P_SPLITLOC, + P_SPLITHAND, + P_NOHALO, + P_SPLITDROP, + P_HIST, /*!< \brief Start of nine patch-size bins. */ + P_STOP = P_HIST + 9, /*!< \brief Start of N_STOP_REASONS bins. */ + P_COUNT = P_STOP + N_STOP_REASONS + }; + unsigned long ct[P_COUNT] = {0}; /*--- One footprint node arriving from a neighbouring rank, to be regrouped by tag. ---*/ struct CInherited { @@ -2251,7 +2122,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, haloVertex[p] = iVertex; } } - unsigned long histogram[9] = {0}; auto markFail = [&](unsigned long f, unsigned short why) { if (!failed[f]) { @@ -2280,8 +2150,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } nodes->SetnChildren_CV(Index_CoarseCV, static_cast(pending[f].size())); Index_CoarseCV++; - nCovered += pending[f].size(); - ((pendingLayers[f] == 1) ? nSemiCV : nFullCV)++; + ct[P_COVERED] += pending[f].size(); + ct[(pendingLayers[f] == 1) ? P_SEMICV : P_FULLCV]++; pending[f].clear(); pendingLayers[f] = 0; @@ -2315,22 +2185,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, return nSeen == layer.size(); }; - /*--- The one test that decides whether a front may advance: is the layer it is about to lay down - * TOPOLOGICALLY IDENTICAL to the layer it is standing on? - * - * "old" and "new" are index-aligned, so phi maps old[k] to new[k], and phi is the extrusion the - * front is proposing. It is a valid layer exactly when phi is an isomorphism of the two induced - * subgraphs AND matches them up one for one: - * - * - same number of cells, which index alignment already gives; - * - every new cell is adjacent to exactly one old cell, and that one is its own preimage; - * - every old cell is adjacent to exactly one new cell, and that one is its own image; - * - the same edges: old[k]-old[l] is an edge if and only if new[k]-new[l] is, which makes the - * edge counts equal and carries connectivity across from the old layer for free. - * - * Nothing else stops a front. There is no cone on how far it may turn and no threshold on how - * stretched the mesh has to be: it runs until it reaches a boundary or until the mesh stops - * offering a clean extrusion, and this is what "stops offering" means. ---*/ + /*--- Is the layer about to be laid topologically identical to the one below? "old" and "new" are + * index-aligned, so phi maps old[k] to new[k]. The layer is valid when phi is an isomorphism of + * the two induced subgraphs: each new cell adjacent to exactly its own preimage, each old cell to + * exactly its own image, and old[k]-old[l] an edge if and only if new[k]-new[l] is. ---*/ auto layerIsIsomorphic = [&](const vector& oldL, const vector& newL) { const auto n = oldL.size(); if (newL.size() != n) return false; @@ -2356,14 +2214,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- The boundary layer of every front. Claiming it before ordinary boundary agglomeration runs is * what keeps the stack aligned: every layer above has exactly the same footprint. ---*/ for (const auto& patch : patches) { - /*--- A one-node patch is allowed to seed a front and marches as a stack one node wide. It looks - * like a poor footprint, but the alternative - leaving it to ordinary agglomeration - is far - * worse: a seed that cannot pair is one whose whole COLUMN then goes unpaved, from the wall to - * wherever the front would have stopped, and those columns land in exactly the places that - * cannot pair for a reason. On the flat plate they were the two nodes at the leading edge, - * where the wall meets the symmetry plane and the marker signatures differ, plus the two - * domain corners: five full-height stripes cut through the paved region, the worst of them - * right at the leading edge. ---*/ + /*--- A one-node patch may seed a front and marches as a stack one node wide. Leaving it to + * ordinary agglomeration is worse: its whole column then goes unpaved, and such seeds sit where + * markers meet, cutting full-height stripes through the paved region. ---*/ bool valid = !patch.empty(); for (auto si : patch) { const auto p = seeds.node[si]; @@ -2394,9 +2247,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, claimed[p] = 1; frontOf[p] = static_cast(f); } - histogram[std::min(layer0.size(), 8)]++; - nStacks++; - nLayers++; + ct[P_HIST + std::min(layer0.size(), 8)]++; + ct[P_STACKS]++; + ct[P_LAYERS]++; emit(f); } @@ -2510,43 +2363,24 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, /*--- Priority order picks the cleanest explanation first: reaching a physical boundary is a * correct, expected stop and takes priority even if some other, non-viable candidate also * happened to be claimed. Only report a collision when no boundary was involved. ---*/ - if (sawBoundary) - markFail(f, STOP_PHYS_BOUNDARY); - else if (sawPartition) { - nNoHalo++; - markFail(f, STOP_PARTITION); - } else if (sawCollision) - markFail(f, STOP_COLLISION); - else if (sawAgglom) - markFail(f, STOP_AGGLOMERATED); - else if (sawGeom) - markFail(f, STOP_GEOMETRY); - else - markFail(f, STOP_NO_NEIGHBOR); + const std::pair why[] = { + {sawBoundary, STOP_PHYS_BOUNDARY}, {sawPartition, STOP_PARTITION}, {sawCollision, STOP_COLLISION}, + {sawAgglom, STOP_AGGLOMERATED}, {sawGeom, STOP_GEOMETRY}, {true, STOP_NO_NEIGHBOR}}; + const auto* hit = why; + while (!hit->first) ++hit; + if (hit->second == STOP_PARTITION) ct[P_NOHALO]++; + markFail(f, hit->second); break; } - CStep s{}; - s.node = best; - s.from = n; - s.key = fine_grid->nodes->GetGlobalIndex(n); - s.score = best_dot; - s.dist = best_len; + CStep s{best, n, fine_grid->nodes->GetGlobalIndex(n), best_dot, best_len, {}}; for (unsigned short d = 0; d < nDim; ++d) s.dir[d] = best_dir[d]; prop[f].push_back(s); } - /*--- A footprint that reaches an interface can be cut by it. If the WHOLE footprint crosses, - * the stack is handed over intact and this front is finished. If only part of it crosses - - * the interface running ALONG the stack rather than across it - the footprint is SPLIT: the - * piece whose successors this rank owns marches on here as a narrower stack, and the rest - * goes to the rank owning the nodes it was reaching for. Both pieces are renamed, because - * they are separate stacks from here on. - * - * Retiring on a cut, which is what this did before, was the expensive part of partitioning: - * an interface that cuts a stack at layer k cuts it at every layer above k as well, so one - * straddle did not cost one layer, it cost the whole remaining column - about a hundred - * nodes each on the flat plate. ---*/ + /*--- An interface can cut a footprint. If all of it crosses, the stack is handed over intact + * and this front is finished. If only part crosses, the footprint is SPLIT: the piece whose + * successors are local marches on here, the rest is handed across, and both are renamed. ---*/ if (failed[f]) { prop[f].clear(); handTo[f].clear(); @@ -2560,7 +2394,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * diagonally does exactly that - and that is not a layer. Drop it and hand over the rest; * the stack still survives on the far side instead of ending here. ---*/ if (!narrow.empty() && !isConnectedLayer(narrow)) { - nSplitDropped += narrow.size(); + ct[P_SPLITDROP] += narrow.size(); narrow.clear(); } @@ -2569,9 +2403,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (narrow.empty()) { prop[f].clear(); } else { - nSplit++; - nSplitLocal += narrow.size(); - nSplitHanded += handTo[f].size(); + ct[P_SPLIT]++; + ct[P_SPLITLOC] += narrow.size(); + ct[P_SPLITHAND] += handTo[f].size(); /*--- Close the coarse CV that is open on the WIDE footprint before narrowing, so that no CV * ever ends up holding two layers of different shape. ---*/ front[f] = narrow; @@ -2595,13 +2429,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, for (unsigned long f = 0; f < front.size(); ++f) { if (!alive[f] || failed[f]) continue; for (const auto& s : prop[f]) { - /*--- A front that has just lost a bid is retiring, so it must not go on to place the rest and - * displace a front that is still healthy. What it placed BEFORE losing does stay in the - * table, and can still cost another front a candidate it would otherwise have won: the - * only way to avoid that entirely is to re-run the contention to a fixed point after every - * retirement. The residual is conservative - it retires a front near a seam one layer - * early, never merges anything it should not - and the seam goes to ordinary agglomeration - * either way, so it is not worth an inner iteration. ---*/ + /*--- A front that has lost a bid is retiring and must not place the rest of its layer and + * displace a healthy front. What it placed before losing stays, which can still cost another + * front a candidate; the residual is conservative and retires a front near a seam one layer + * early, and the seam goes to ordinary agglomeration either way. ---*/ if (failed[f]) break; if (bidIdx[s.node] == NOBID) { @@ -2660,16 +2491,18 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (failed[f]) { /*--- Nothing to give back: a bid only becomes a claim on acceptance below. ---*/ alive[f] = 0; - stopCounts[failReason[f]]++; + ct[P_STOP + failReason[f]]++; /*--- One layer short of a full block at the top: take what is buffered as its own coarse CV * rather than dropping it back to ordinary agglomeration. ---*/ emit(f); continue; } - /*--- Accept. The direction is blended rather than replaced, and it is the FRONT's direction, - * updated once from the mean of the steps its nodes just took, not one direction per node - * free to wander off on its own. ---*/ + /*--- (d) Hand stacks across partition interfaces. A front that runs into the halo cannot go on + * here, so its footprint is sent to the owning rank, which picks the stack up and carries on. + * What crosses is the footprint, not a coarse CV, so both halves keep the same shape. It travels + * the reverse of the usual halo direction: packed against the RECEIVE marker and sent to the + * rank that marker receives from. ---*/ su2double mean[MAXNDIM] = {0.0}; for (const auto& s : prop[f]) for (unsigned short d = 0; d < nDim; ++d) mean[d] += s.dir[d]; @@ -2689,28 +2522,18 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } front[f] = std::move(newLayer); depth[f]++; - nLayers++; + ct[P_LAYERS]++; pending[f].insert(pending[f].end(), front[f].begin(), front[f].end()); pendingLayers[f]++; if (pendingLayers[f] >= nBlock[f]) emit(f); } - /*============================================================================================== - * (d) Hand stacks across partition interfaces. - * - * A front that has run into the halo cannot go on here: those nodes belong to another rank, and - * their parent is that rank's to assign. Without this the stack simply ended at the interface, - * and on four ranks that was 70 of 71 fronts - the paved fraction of the flat plate fell from - * 99% to 48% purely because of where the partition happened to cut. - * - * Instead the footprint is sent to the owner, which picks the stack up and carries on. What - * crosses is not a coarse CV - a CV belongs wholly to one rank - but the FOOTPRINT, so the two - * halves of the stack stay the same shape and the coarse grid reads the same across the seam. - * The handover travels the reverse of the usual halo direction: a node this rank sees as halo is - * one the neighbour owns, so it is packed against the RECEIVE marker and sent to the rank this - * marker normally receives from. - *============================================================================================*/ + /*--- (d) Hand stacks across partition interfaces. A front that runs into the halo cannot go on + * here, so the footprint is sent to the owning rank, which picks the stack up and carries on. + * What crosses is the footprint, not a coarse CV, so both halves keep the same shape. It travels + * the reverse of the usual halo direction: packed against the RECEIVE marker and sent to the + * rank that marker receives from. ---*/ for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { if (!((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0))) continue; @@ -2758,10 +2581,10 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, for (unsigned long f = 0; f < front.size(); ++f) { if (handTo[f].empty()) continue; handTo[f].clear(); - nHandedOut++; + ct[P_HANDOUT]++; if (keepLocal[f]) continue; alive[f] = 0; - stopCounts[STOP_PARTITION]++; + ct[P_STOP + STOP_PARTITION]++; emit(f); } @@ -2796,8 +2619,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, claimed[p] = 1; frontOf[p] = static_cast(nf); } - nLayers++; - nHandedIn++; + ct[P_LAYERS]++; + ct[P_HANDIN]++; if (pendingLayers[nf] >= nBlock[nf]) emit(nf); } i = j; @@ -2809,13 +2632,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * with a parent index that was never assigned. ---*/ for (unsigned long f = 0; f < front.size(); ++f) emit(f); - /*--- How far each front actually got. This is the number to watch when the paved region does not - * look like a front: fronts that all reach the same height leave a flat interface with ordinary - * agglomeration, and a spread here is that interface coming out as a staircase instead. - * - * A rank with no fronts of its own must not drag the reported minimum to zero: leaving dmin at - * its sentinel keeps it out of the MPI_MIN below, so the range describes the fronts that exist - * rather than the ranks that have none. ---*/ + /*--- How far each front got. Fronts reaching the same height leave a flat interface with ordinary + * agglomeration; a spread here means that interface came out as a staircase. A rank with no fronts + * leaves dmin at its sentinel so it stays out of the MPI_MIN below. ---*/ unsigned long dmin = std::numeric_limits::max(), dmax = 0; for (unsigned long f = 0; f < front.size(); ++f) { if (front[f].empty()) continue; @@ -2823,57 +2642,41 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, dmax = std::max(dmax, depth[f]); } - /*--- Summary over all ranks. Reporting rank 0's own fronts makes a partitioned run look like a - * fraction of the mesh it is not, and hides how much of the layer the partitioning cost: a front - * stops at the partition, so the number of nodes left to ordinary agglomeration is the number to - * watch when adding ranks. Every rank must reach these collectives. ---*/ - unsigned long nSeedNodes = seeds.node.size(); - unsigned long local[13] = {nSeedNodes, nStacks, nLayers, nCovered, nSemiCV, nFullCV, nHandedOut, - nHandedIn, nSplit, nSplitLocal, nSplitHanded, nNoHalo, nSplitDropped}; - unsigned long total[13] = {0}; - SU2_MPI::Allreduce(local, total, 13, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - - unsigned long localCV = Index_CoarseCV - starting_Index_CoarseCV, totalCV = 0; - SU2_MPI::Allreduce(&localCV, &totalCV, 1, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); + /*--- Summed over all ranks: rank 0's own fronts would make a partitioned run look like a fraction of + * the mesh it is not. Every rank must reach these collectives. ---*/ + unsigned long tot[P_COUNT] = {0}; + SU2_MPI::Allreduce(ct, tot, P_COUNT, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - unsigned long histTotal[9] = {0}; - SU2_MPI::Allreduce(histogram, histTotal, 9, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - - vector stopTotal(N_STOP_REASONS, 0); - SU2_MPI::Allreduce(stopCounts.data(), stopTotal.data(), N_STOP_REASONS, MPI_UNSIGNED_LONG, MPI_SUM, - SU2_MPI::GetComm()); + unsigned long pair[2] = {Index_CoarseCV - starting_Index_CoarseCV, seeds.node.size()}, pairTot[2] = {0}; + SU2_MPI::Allreduce(pair, pairTot, 2, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); unsigned long depthMin = 0, depthMax = 0; SU2_MPI::Allreduce(&dmin, &depthMin, 1, MPI_UNSIGNED_LONG, MPI_MIN, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&dmax, &depthMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); if (depthMin == std::numeric_limits::max()) depthMin = 0; /*--- No fronts anywhere. ---*/ - if (rank == MASTER_NODE) { - cout << " Paving fronts: " << total[1] << " fronts from " << total[0] << " seed nodes, patch sizes "; - for (unsigned s = 1; s <= 8; ++s) - if (histTotal[s] > 0) cout << s << "x" << histTotal[s] << " "; - cout << "\n Coarse CVs from fronts: " << totalCV << " covering " << total[3] << " nodes in " << total[2] - << " layers, front depth " << depthMin << " to " << depthMax; - if (total[4] + total[5] > 0) - cout << "\n Coarse CVs by depth: " << total[5] << " two layers deep, " << total[4] - << " one layer (top of a stack)"; - if (total[6] + total[7] > 0) - cout << "\n Stacks handed across partitions: " << total[6] << " sent, " << total[7] << " picked up"; - if (total[8] > 0) - cout << "\n Footprints split at partitions: " << total[8] << " cut by an interface (" << total[9] - << " nodes marching on here, " << total[10] << " handed across)"; - if (total[11] + total[12] > 0) - cout << "\n Stacks lost at partitions: " << total[11] << " blocked with nowhere to hand to, " << total[12] - << " nodes in split pieces that came apart"; - /*--- Why each front stopped. Reaching a boundary is the one correct stop; everything else is the - * mesh failing to offer a layer topologically identical to the current one, broken down by how - * it failed. COLLISION and PINCH are two fronts, or two nodes of one front, reaching for the - * same node; TOPOLOGY is a layer that was claimable but not an extrusion. ---*/ - cout << "\n Front advance stopped due to: physical-boundary " << stopTotal[STOP_PHYS_BOUNDARY] << ", partition " - << stopTotal[STOP_PARTITION] << ", front-collision " << stopTotal[STOP_COLLISION] << ", pinch " - << stopTotal[STOP_PINCH] << ", already-agglomerated " << stopTotal[STOP_AGGLOMERATED] << ", dead-end " - << stopTotal[STOP_NO_NEIGHBOR] << ", topology " << stopTotal[STOP_TOPOLOGY] << ", geometry " - << stopTotal[STOP_GEOMETRY]; - cout << endl; - } + if (rank != MASTER_NODE) return; + + cout << " Paving fronts: " << tot[P_STACKS] << " fronts from " << pairTot[1] << " seed nodes, patch sizes "; + for (unsigned n = 1; n <= 8; ++n) + if (tot[P_HIST + n] > 0) cout << n << "x" << tot[P_HIST + n] << " "; + cout << "\n Coarse CVs from fronts: " << pairTot[0] << " covering " << tot[P_COVERED] << " nodes in " + << tot[P_LAYERS] << " layers, front depth " << depthMin << " to " << depthMax; + if (tot[P_SEMICV] + tot[P_FULLCV] > 0) + cout << "\n Coarse CVs by depth: " << tot[P_FULLCV] << " two layers deep, " << tot[P_SEMICV] + << " one layer (top of a stack)"; + if (tot[P_HANDOUT] + tot[P_HANDIN] > 0) + cout << "\n Stacks handed across partitions: " << tot[P_HANDOUT] << " sent, " << tot[P_HANDIN] << " picked up"; + if (tot[P_SPLIT] > 0) + cout << "\n Footprints split at partitions: " << tot[P_SPLIT] << " cut by an interface (" << tot[P_SPLITLOC] + << " nodes marching on here, " << tot[P_SPLITHAND] << " handed across)"; + if (tot[P_NOHALO] + tot[P_SPLITDROP] > 0) + cout << "\n Stacks lost at partitions: " << tot[P_NOHALO] << " blocked with nowhere to hand to, " + << tot[P_SPLITDROP] << " nodes in split pieces that came apart"; + + static const char* stopName[N_STOP_REASONS] = {"physical-boundary", "partition", "front-collision", "pinch", + "already-agglomerated", "dead-end", "topology", "geometry"}; + cout << "\n Front advance stopped due to:"; + for (unsigned n = 0; n < N_STOP_REASONS; ++n) cout << (n ? ", " : " ") << stopName[n] << " " << tot[P_STOP + n]; + cout << endl; } diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index a0e9fa26b02..e6292b4c10c 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -1468,13 +1468,6 @@ void CSysMatrix::BuildLineletPreconditioner(const CGeometry* geometr } END_SU2_OMP_SAFE_GLOBAL_ACCESS - /*--- A rank whose part of the mesh holds no solid wall gets no linelets, and the working vectors - * above were deliberately not allocated for it. It still reaches this function, because the - * linear solver is collective, so it has to leave before indexing them. Partitioning a mesh - * over enough ranks eventually gives one of them no wall, which is why this only ever showed up - * beyond a couple of ranks. ---*/ - if (LineletUpper.empty()) return; - SU2_OMP_FOR_STAT(1) for (int iThread = 0; iThread < nThreads; ++iThread) { const auto size = CGeometry::CLineletInfo::MAX_LINELET_POINTS; diff --git a/Common/src/linear_algebra/CSysSolve.cpp b/Common/src/linear_algebra/CSysSolve.cpp index 236dce58271..3c9176d8716 100644 --- a/Common/src/linear_algebra/CSysSolve.cpp +++ b/Common/src/linear_algebra/CSysSolve.cpp @@ -1435,34 +1435,6 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con const bool nested = SetupInnerSolver(KindSolver, config); - /*--- Decide whether to rebuild the preconditioner for this solve. - * - * On coarse multigrid levels the Jacobian changes little between smoothing sweeps, yet the - * factorization is rebuilt from scratch on every one of them. Profiling a V-cycle shows the - * ILU build is the single most expensive zone in the cycle, so MG_COARSE_PREC_FREEZE lets a - * factorization be reused for several consecutive solves. The factorization itself lives in - * the CSysMatrix, which outlives the CPreconditioner object created below, so not calling - * Build() is all that is needed to reuse it. - * - * Restricted to the standard solver mode (mesh deformation and gradient smoothing are left - * alone), and to the coarse levels: the finest grid always rebuilds, because its preconditioner - * drives the outer nonlinear convergence and freezing it measured worth far less than freezing - * the coarse levels. The first solve on this instance always builds (count 0), which matters - * because the factorization is otherwise uninitialized. - * - * The decision is taken by one thread and read by all of them, because Build() is internally - * OpenMP-parallel and every thread must make the same choice. ---*/ - - BEGIN_SU2_OMP_SAFE_GLOBAL_ACCESS { - unsigned long freeze = 1; - if (lin_sol_mode == LINEAR_SOLVER_MODE::STANDARD && geometry != nullptr && geometry->GetMGLevel() != MESH_0) { - freeze = std::max(1, config->GetMGOptions().MG_Coarse_Prec_Freeze); - } - buildPrecThisSolve = (precSolveCount % freeze == 0); - precSolveCount++; - } - END_SU2_OMP_SAFE_GLOBAL_ACCESS - /*--- Stop the recording for the linear solver ---*/ bool TapeActive = NO; @@ -1500,7 +1472,7 @@ unsigned long CSysSolve::Solve(CSysMatrix& Jacobian, con const auto kindPrec = static_cast(KindPrecond); auto* normal_prec = CPreconditioner::Create(kindPrec, Jacobian, geometry, config); - if (buildPrecThisSolve) normal_prec->Build(); + normal_prec->Build(); CPreconditioner* nested_prec = nullptr; if (nested) { From 625317b64c8c1e2a0b610c5a4bfb259f40dbe7b3 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 19:00:44 +0200 Subject: [PATCH 47/54] cleanup diagnostics --- .../include/geometry/CMultiGridGeometry.hpp | 16 ---- Common/src/geometry/CMultiGridGeometry.cpp | 89 ++++--------------- 2 files changed, 18 insertions(+), 87 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index 72a7ddb4592..03b0951a188 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -115,22 +115,6 @@ class CMultiGridGeometry final : public CGeometry { */ CNodeStiffness ComputeNodeStiffness(const CGeometry* fine_grid) const; - /*! - * \brief Why a front stopped advancing. Either it reached a boundary, or it could not lay a layer - * isomorphic to the current one; every reason below is one of those two. - */ - enum { - STOP_PHYS_BOUNDARY = 0, /*!< \brief Reached a boundary, the expected stop. */ - STOP_PARTITION, /*!< \brief Reached a partition interface. */ - STOP_COLLISION, /*!< \brief Lost a candidate to another front. */ - STOP_PINCH, /*!< \brief Two nodes of this front wanted the same successor. */ - STOP_AGGLOMERATED, /*!< \brief Ran into nodes an earlier phase had taken. */ - STOP_NO_NEIGHBOR, /*!< \brief A front node had no free neighbour left. */ - STOP_TOPOLOGY, /*!< \brief The next layer was not isomorphic to the current one. */ - STOP_GEOMETRY, /*!< \brief A node of the next layer failed GeometricalCheck. */ - N_STOP_REASONS - }; - /*! * \brief Boundary nodes that seed a front, with the direction each starts marching in. */ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 3f3889c43b5..c88b3456cf2 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -2045,10 +2045,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, vector claimed(nPointFine, 0); /*--- Confirmed owner of a claimed node, -1 while free. Only ever written when a layer is accepted, * so a bid that is still being contested never appears here. ---*/ - vector frontOf(nPointFine, -1); vector failed; - vector failReason; /*--- Set when a front hands only PART of its footprint over and goes on marching here with what is * left of it, so the retirement pass at the end of the round knows not to kill it. ---*/ @@ -2092,11 +2090,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, P_SPLIT, P_SPLITLOC, P_SPLITHAND, - P_NOHALO, P_SPLITDROP, - P_HIST, /*!< \brief Start of nine patch-size bins. */ - P_STOP = P_HIST + 9, /*!< \brief Start of N_STOP_REASONS bins. */ - P_COUNT = P_STOP + N_STOP_REASONS + P_HIST, /*!< \brief Start of nine patch-size bins. */ + P_COUNT = P_HIST + 9 }; unsigned long ct[P_COUNT] = {0}; @@ -2123,12 +2119,7 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } } - auto markFail = [&](unsigned long f, unsigned short why) { - if (!failed[f]) { - failed[f] = 1; - failReason[f] = why; - } - }; + auto markFail = [&](unsigned long f) { failed[f] = 1; }; /*--- How many fine layers the next coarse CV of this front holds: always two, so the stack coarsens * by the same factor along the marching direction as the footprint does across it. The only @@ -2245,7 +2236,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto f = addFront(layer0, n0, frontTag + 1, 1); for (auto p : layer0) { claimed[p] = 1; - frontOf[p] = static_cast(f); } ct[P_HIST + std::min(layer0.size(), 8)]++; ct[P_STACKS]++; @@ -2265,7 +2255,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (aliveGlobal == 0) break; failed.assign(front.size(), 0); - failReason.assign(front.size(), 0); keepLocal.assign(front.size(), 0); handTag.assign(front.size(), 0); for (const auto& b : bids) bidIdx[b.node] = NOBID; @@ -2287,8 +2276,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * here, but it is where the stack would go next, so it is what gets handed over. ---*/ auto bestHalo = NO_POINT; su2double bestHalo_dot = -2.0; - bool sawCollision = false, sawBoundary = false, sawAgglom = false, sawPartition = false; - bool sawGeom = false; for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(n); ++iNeigh) { const auto jPoint = fine_grid->nodes->GetPoint(n, iNeigh); @@ -2310,7 +2297,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * own, rather than being skipped silently and leaving some other candidate to explain a * stop that was really the partitioning. ---*/ if (!fine_grid->nodes->GetDomain(jPoint)) { - sawPartition = true; /*--- Held as a handover candidate, subject to the same admissibility the owner would * apply anyway; whether it is still free is the owner's to decide. ---*/ if (dot > bestHalo_dot && !(onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) && @@ -2320,28 +2306,13 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, } continue; } - if (fine_grid->nodes->GetAgglomerate(jPoint)) { - /*--- A node another front has already turned into a coarse CV also reads as agglomerated, - * so ask frontOf first: that is the fronts meeting, not an earlier phase. ---*/ - if ((frontOf[jPoint] >= 0) && (frontOf[jPoint] != static_cast(f))) - sawCollision = true; - else if (frontOf[jPoint] < 0) - sawAgglom = true; - continue; - } - if (claimed[jPoint]) { - if (frontOf[jPoint] != static_cast(f)) sawCollision = true; - continue; - } - - if (onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) { - sawBoundary = true; - continue; - } - if (!GeometricalCheck(jPoint, fine_grid, config)) { - sawGeom = true; - continue; - } + /*--- Taken by an earlier phase or by another front's coarse CV. ---*/ + if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; + /*--- Already bid for this round. ---*/ + if (claimed[jPoint]) continue; + /*--- The step runs into a boundary, or the node would make an unusable coarse cell. ---*/ + if (onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) continue; + if (!GeometricalCheck(jPoint, fine_grid, config)) continue; if (dot > best_dot) { best_dot = dot; @@ -2359,17 +2330,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, continue; } + /*--- No admissible successor: a boundary, a partition, another front, or unusable mesh. ---*/ if (best == NO_POINT) { - /*--- Priority order picks the cleanest explanation first: reaching a physical boundary is a - * correct, expected stop and takes priority even if some other, non-viable candidate also - * happened to be claimed. Only report a collision when no boundary was involved. ---*/ - const std::pair why[] = { - {sawBoundary, STOP_PHYS_BOUNDARY}, {sawPartition, STOP_PARTITION}, {sawCollision, STOP_COLLISION}, - {sawAgglom, STOP_AGGLOMERATED}, {sawGeom, STOP_GEOMETRY}, {true, STOP_NO_NEIGHBOR}}; - const auto* hit = why; - while (!hit->first) ++hit; - if (hit->second == STOP_PARTITION) ct[P_NOHALO]++; - markFail(f, hit->second); + markFail(f); break; } @@ -2446,14 +2409,12 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto g = bidOwner[k]; /*--- Two nodes of the SAME front reaching for one successor is a pinch: the layer would come * out narrower than the front, which all-or-nothing does not allow. ---*/ - const unsigned short why = (g == f) ? STOP_PINCH : STOP_COLLISION; - if (better(s, bids[k])) { - markFail(g, why); + markFail(g); bids[k] = s; bidOwner[k] = f; } else { - markFail(f, why); + markFail(f); } /*--- A head-on meeting stops BOTH fronts. Letting the winner carry on through the seam would @@ -2461,8 +2422,8 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * shows up in the coarse grid as one stack overshooting the other. A glancing contact * (directions not opposed) is not a meeting and only costs the loser. ---*/ if ((g != f) && (GeometryToolbox::DotProduct(nDim, dirNow[f].data(), dirNow[g].data()) < 0.0)) { - markFail(f, STOP_COLLISION); - markFail(g, STOP_COLLISION); + markFail(f); + markFail(g); } } } @@ -2482,16 +2443,12 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if ((k != NOBID) && (bidOwner[k] == f)) newLayer.push_back(s.node); } /*--- Every bid of a front that was not marked failed must have been granted. ---*/ - if (newLayer.size() != front[f].size()) - markFail(f, STOP_COLLISION); - else if (!layerIsIsomorphic(front[f], newLayer)) - markFail(f, STOP_TOPOLOGY); + if ((newLayer.size() != front[f].size()) || !layerIsIsomorphic(front[f], newLayer)) markFail(f); } if (failed[f]) { /*--- Nothing to give back: a bid only becomes a claim on acceptance below. ---*/ alive[f] = 0; - ct[P_STOP + failReason[f]]++; /*--- One layer short of a full block at the top: take what is buffered as its own coarse CV * rather than dropping it back to ordinary agglomeration. ---*/ emit(f); @@ -2518,7 +2475,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, for (auto p : newLayer) { claimed[p] = 1; - frontOf[p] = static_cast(f); } front[f] = std::move(newLayer); depth[f]++; @@ -2584,7 +2540,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, ct[P_HANDOUT]++; if (keepLocal[f]) continue; alive[f] = 0; - ct[P_STOP + STOP_PARTITION]++; emit(f); } @@ -2617,7 +2572,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const auto nf = addFront(layer0, d0, inherited[i].tag, blockFor(layer0)); for (auto p : layer0) { claimed[p] = 1; - frontOf[p] = static_cast(nf); } ct[P_LAYERS]++; ct[P_HANDIN]++; @@ -2670,13 +2624,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, if (tot[P_SPLIT] > 0) cout << "\n Footprints split at partitions: " << tot[P_SPLIT] << " cut by an interface (" << tot[P_SPLITLOC] << " nodes marching on here, " << tot[P_SPLITHAND] << " handed across)"; - if (tot[P_NOHALO] + tot[P_SPLITDROP] > 0) - cout << "\n Stacks lost at partitions: " << tot[P_NOHALO] << " blocked with nowhere to hand to, " - << tot[P_SPLITDROP] << " nodes in split pieces that came apart"; - - static const char* stopName[N_STOP_REASONS] = {"physical-boundary", "partition", "front-collision", "pinch", - "already-agglomerated", "dead-end", "topology", "geometry"}; - cout << "\n Front advance stopped due to:"; - for (unsigned n = 0; n < N_STOP_REASONS; ++n) cout << (n ? ", " : " ") << stopName[n] << " " << tot[P_STOP + n]; + if (tot[P_SPLITDROP] > 0) cout << "\n Nodes lost in split pieces that came apart: " << tot[P_SPLITDROP]; cout << endl; } From 12516997fb590c904ff846b6073ea9c3cc358082 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 19:46:28 +0200 Subject: [PATCH 48/54] cleanup config option --- Common/include/option_structure.hpp | 3 --- Common/src/CConfig.cpp | 4 ---- Common/src/geometry/CMultiGridGeometry.cpp | 16 ++++------------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Common/include/option_structure.hpp b/Common/include/option_structure.hpp index 9440ecc5e0b..a112ea431ad 100644 --- a/Common/include/option_structure.hpp +++ b/Common/include/option_structure.hpp @@ -1125,9 +1125,6 @@ struct CMGOptions { bool MG_Smooth_Output{false}; /*!< \brief Output compact per-cycle smoothing summary. */ su2double MG_Smooth_StagnationTol{0.0}; /*!< \brief Stagnation early exit: stop if current_rms >= prev_rms * tol. 0 = disabled. */ bool MG_Implicit_Lines{false}; /*!< \brief Enable implicit-lines agglomeration from walls. */ - unsigned long MG_Implicit_Lines_Max_Group{0}; /*!< \brief Max number of parallel implicit lines merged into one coarse - CV tangential to the wall. 0 = dimension-appropriate default - (2 in 2D, 4 in 3D). See CMultiGridGeometry::AgglomerateImplicitLines. */ unsigned long MG_Startup_Iter{100}; /*!< \brief Iterations per mesh during FMG startup, and the length of each level's CFL ramp. 0 = no iteration budget. */ su2double MG_Startup_Convergence{-2.0}; /*!< \brief FMG: orders of magnitude (log10) that CONV_FIELD must drop on the active level before promoting to the next finer one. Negative is a diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index ee2036c598f..e9bcd8d0791 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2069,10 +2069,6 @@ void CConfig::SetConfig_Options() { addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); - /*!\brief MG_IMPLICIT_LINES_MAX_GROUP\n DESCRIPTION: Maximum number of parallel implicit lines merged tangential to - * the wall into one coarse CV (2D: always 2; 3D: e.g. 4 for a wall quad/hex corner, 3 for a triangular prism apex). - * 0 uses the dimension-appropriate default (2 in 2D, 4 in 3D). DEFAULT: 0 \ingroup Config*/ - addUnsignedLongOption("MG_IMPLICIT_LINES_MAX_GROUP", MGOptions.MG_Implicit_Lines_Max_Group, 0); /*!\brief MG_STARTUP_ITER\n DESCRIPTION: Max number of iterations spent on each mesh during the Full * Multigrid (FMG) startup phase. DEFAULT: 100 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_ITER", MGOptions.MG_Startup_Iter, 100); diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index c88b3456cf2..08fdaa817d7 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1799,8 +1799,7 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront * exclusive, which choosing neighbours per seed would not. It needs only point connectivity, so it * works on every multigrid level. This patch alone fixes the footprint of the stack above it. ---*/ const auto nSeeds = seeds.node.size(); - unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; - if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; + const unsigned long max_group = (nDim == 2) ? 2 : 4; /*--- Marker signature of each seed, as a bitmask over the physical markers. Seeds may only be * matched when these agree, so a patch never straddles a change of boundary condition - the rule @@ -1963,11 +1962,6 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, * only ever RANKS candidates, it never rejects one, so this is a preference and not a limit. ---*/ constexpr passivedouble DIR_BLEND = 0.5; - /*--- How many parallel implicit lines may be merged tangential to the wall into one coarse CV. - * 0 selects the dimension-appropriate default: 2 in 2D, 4 in 3D (a wall quad/hex corner). ---*/ - unsigned long max_group = config->GetMGOptions().MG_Implicit_Lines_Max_Group; - if (max_group == 0) max_group = (nDim == 2) ? 2 : 4; - const auto stiff = ComputeNodeStiffness(fine_grid); /*--- PHASE 1. SeedFrontNodes must be reached by every rank, including one that owns no boundary: @@ -2121,11 +2115,9 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, auto markFail = [&](unsigned long f) { failed[f] = 1; }; - /*--- How many fine layers the next coarse CV of this front holds: always two, so the stack coarsens - * by the same factor along the marching direction as the footprint does across it. The only - * exception is a footprint already so wide that a second layer would exceed the agglomeration - * size limit, which can only happen if MG_IMPLICIT_LINES_MAX_GROUP was raised past the - * dimension's default. ---*/ + /*--- How many fine layers the next coarse CV of this front holds: two, so the stack coarsens by the + * same factor along the marching direction as the footprint does across it. One only if a second + * layer would exceed the agglomeration size limit. ---*/ auto blockFor = [&](const vector& layer) -> unsigned long { return (layer.size() * 2 > static_cast(maxAgglomSize)) ? 1 : 2; }; From 6008bd3a0a83762f7ab45b745ebe24c2a3cd2341 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Sun, 6 Sep 2026 22:06:08 +0200 Subject: [PATCH 49/54] cleanup comments --- .../src/integration/CMultiGridIntegration.cpp | 44 +++---------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index c06fb87994d..7e60d634c7a 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -685,19 +685,6 @@ void CMultiGridIntegration::MultiGrid_Cycle(CGeometry ****geometry, SU2_OMP_SAFE_GLOBAL_ACCESS(config->SetKind_TimeIntScheme(EULER_IMPLICIT);) } - /*--- NOTE: the coarse-grid residual computed just above is evaluated at the restricted - * solution, i.e. at exactly the state the first pre-smoothing sweep of the recursive call - * below re-evaluates it at, so it looks like that sweep could reuse LinSysRes (and, if the - * Jacobian were assembled here, the Jacobian too) and skip its own Preprocessing and - * Space_Integration. It cannot, as things stand: Space_Integration is not a pure producer of - * LinSysRes/Jacobian. BC_Sym_Plane (which serves both SYMMETRY_PLANE and EULER_WALL) also - * projects Res_TruncError and Solution_Old onto the wall tangent plane, and in the current - * ordering that projection is what makes the FAS forcing term written by SetForcing_Term - * below, and the Solution_Old written by Set_OldSolution, wall-consistent before their first - * use. Reusing the residual moves both projections to the wrong side of the writes. - * Factoring those side effects out of Space_Integration would make the reuse safe and save - * one full residual evaluation per coarse level per cycle. ---*/ - /*--- Recursive call to MultiGrid_Cycle (this routine). ---*/ /*--- Execute multigrid cycles sequentially to ensure deterministic recursion order ---*/ /*--- This prevents accumulation of floating-point variations across recursive calls ---*/ @@ -974,16 +961,13 @@ void CMultiGridIntegration::GetProlongated_Correction(unsigned short RunTime_EqS } } - /*--- MPI the set solution old. Required: the loop above only writes domain points, and - * ProlongateField below injects from every coarse point including halos in order to fill the - * fine-grid halo entries of the correction. ---*/ + /*--- MPI the set solution old. ---*/ sol_coarse->InitiateComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); sol_coarse->CompleteComms(geo_coarse, config, MPI_QUANTITIES::SOLUTION_OLD); - /*--- Interpolate the coarse-grid correction (held in Solution_Old) onto the fine - * grid and store it in LinSysRes, which SetProlongated_Correction then damps - * and adds to the fine-grid solution. ---*/ + /*--- Interpolate the coarse-grid correction onto the fine + * grid and store in LinSysRes. ---*/ ProlongateField(geo_coarse, [&](unsigned long iPoint) { return sol_coarse->GetNodes()->GetSolution_Old(iPoint); }, @@ -1003,10 +987,6 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ const unsigned short nVar = solver->GetnVar(); - /*--- Seeded over all points, halos included: the restore loop below reads Residual_Old at the - * vertices of the physical markers, and on a partitioned mesh some of those are halo points - * owned by another rank. ---*/ - SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPoint(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geometry->GetnPoint(); iPoint++) { const auto* Residual_Old = solver->LinSysRes.GetBlock(iPoint); @@ -1019,10 +999,7 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ for (auto iSmooth = 0u; iSmooth < val_nSmooth; iSmooth++) { - /*--- Loop over the domain points (sum the residuals of direct neighbors). - * Halo points are deliberately not smoothed here: their own neighbor stencil is incomplete - * on this rank, so the average would be meaningless, and the halo exchange at the end of - * each sweep overwrites them with the value their owner computed anyway. ---*/ + /*--- Loop over the domain points, exclude halo points ---*/ SU2_OMP_FOR_STAT(roundUpDiv(geometry->GetnPointDomain(), omp_get_num_threads())) for (auto iPoint = 0ul; iPoint < geometry->GetnPointDomain(); ++iPoint) { @@ -1053,17 +1030,7 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } END_SU2_OMP_FOR - /*--- Restore original residuals (without average) at physical boundary points. - * - * SEND_RECEIVE is excluded: carrying such a marker does not put a point on a boundary, it - * only records that the point is mirrored on another rank. Restoring those points froze the - * correction on the whole send fringe, which is exactly the ring of domain points that have - * a halo neighbour, so the smoothing this function applied depended on where the mesh - * happened to be partitioned rather than on the geometry alone. - * - * Note this removes one source of rank-dependence, not all of them: the coarse grids are - * agglomerated per rank, so the multigrid operator itself still differs between partition - * counts and a run on 1 and on N ranks is not expected to match bit for bit. ---*/ + /*--- Restore original residuals at physical boundary points. ---*/ for (auto iMarker = 0u; iMarker < geometry->GetnMarker(); iMarker++) { if ((config->GetMarker_All_KindBC(iMarker) != INTERNAL_BOUNDARY) && @@ -1366,7 +1333,6 @@ void CMultiGridIntegration::SetRestricted_Gradient(unsigned short RunTime_EqSyst SU2_OMP_FOR_STAT(roundUpDiv(geo_coarse->GetnPoint(), omp_get_num_threads())) for (auto Point_Coarse = 0ul; Point_Coarse < geo_coarse->GetnPoint(); Point_Coarse++) { - /*--- Row-major scratch plus the row pointers SetGradient expects. ---*/ su2double GradientData[MAXNVAR][MAXNDIM] = {{0.0}}; su2double* Gradient[MAXNVAR]; for (auto iVar = 0u; iVar < nVar; iVar++) Gradient[iVar] = GradientData[iVar]; From 5de0a00de76b31f8394c98c3a4b461e2f7378348 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Mon, 7 Sep 2026 17:12:05 +0200 Subject: [PATCH 50/54] properly write the implicit line agglomeration info --- .../include/geometry/CMultiGridGeometry.hpp | 8 ++++- Common/src/geometry/CMultiGridGeometry.cpp | 33 ++++++++++--------- SU2_CFD/src/drivers/CDriver.cpp | 9 ++++- .../src/integration/CMultiGridIntegration.cpp | 11 +------ 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index 03b0951a188..eee6038071e 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -91,8 +91,11 @@ class CMultiGridGeometry final : public CGeometry { * \param[in,out] Index_CoarseCV - Current coarse CV index, incremented as new coarse CVs are created. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Configuration. + * \param[in] iMesh - Multigrid level being built, used to label the summary. + * \return Summary of the paving, empty except on the master rank. */ - void AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config); + string AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, + unsigned short iMesh); /*! * \brief Weakest and strongest dual-grid coupling at each node, and the neighbour across the @@ -146,6 +149,9 @@ class CMultiGridGeometry final : public CGeometry { const CConfig* config, const vector& mixedBC) const; public: + /*!< \brief Paving summary for this level. */ + string pavingReport; + /*--- This is to suppress Woverloaded-virtual, omitting it has no negative impact. ---*/ using CGeometry::SetBoundControlVolume; using CGeometry::SetControlVolume; diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 08fdaa817d7..d8eaaf9e98b 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -94,7 +94,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un * CVs it creates occupy [firstLineCV, endLineCV), which is how the repair passes identify them. ---*/ const auto firstLineCV = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { - AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config); + pavingReport = AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, iMesh); } const auto endLineCV = Index_CoarseCV; @@ -1939,8 +1939,8 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront return groups; } -void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, - const CConfig* config) { +string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, + const CConfig* config, unsigned short iMesh) { /*--- Paving by advancing fronts. Each boundary patch from PHASE 1 rises into the domain keeping its * footprint: every front node picks a successor and the front advances only if all succeed, so the * layers of a stack are congruent and a coarse CV never spans two patches. A front stops at a @@ -2601,21 +2601,24 @@ void CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, SU2_MPI::Allreduce(&dmax, &depthMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); if (depthMin == std::numeric_limits::max()) depthMin = 0; /*--- No fronts anywhere. ---*/ - if (rank != MASTER_NODE) return; + if (rank != MASTER_NODE) return {}; - cout << " Paving fronts: " << tot[P_STACKS] << " fronts from " << pairTot[1] << " seed nodes, patch sizes "; + stringstream out; + out << " MG level " << iMesh << " paving: " << tot[P_STACKS] << " fronts from " << pairTot[1] + << " seed nodes, patch sizes "; for (unsigned n = 1; n <= 8; ++n) - if (tot[P_HIST + n] > 0) cout << n << "x" << tot[P_HIST + n] << " "; - cout << "\n Coarse CVs from fronts: " << pairTot[0] << " covering " << tot[P_COVERED] << " nodes in " - << tot[P_LAYERS] << " layers, front depth " << depthMin << " to " << depthMax; + if (tot[P_HIST + n] > 0) out << n << "x" << tot[P_HIST + n] << " "; + out << "\n Coarse CVs from fronts: " << pairTot[0] << " covering " << tot[P_COVERED] << " nodes in " << tot[P_LAYERS] + << " layers, front depth " << depthMin << " to " << depthMax; if (tot[P_SEMICV] + tot[P_FULLCV] > 0) - cout << "\n Coarse CVs by depth: " << tot[P_FULLCV] << " two layers deep, " << tot[P_SEMICV] - << " one layer (top of a stack)"; + out << "\n Coarse CVs by depth: " << tot[P_FULLCV] << " two layers deep, " << tot[P_SEMICV] + << " one layer (top of a stack)"; if (tot[P_HANDOUT] + tot[P_HANDIN] > 0) - cout << "\n Stacks handed across partitions: " << tot[P_HANDOUT] << " sent, " << tot[P_HANDIN] << " picked up"; + out << "\n Stacks handed across partitions: " << tot[P_HANDOUT] << " sent, " << tot[P_HANDIN] << " picked up"; if (tot[P_SPLIT] > 0) - cout << "\n Footprints split at partitions: " << tot[P_SPLIT] << " cut by an interface (" << tot[P_SPLITLOC] - << " nodes marching on here, " << tot[P_SPLITHAND] << " handed across)"; - if (tot[P_SPLITDROP] > 0) cout << "\n Nodes lost in split pieces that came apart: " << tot[P_SPLITDROP]; - cout << endl; + out << "\n Footprints split at partitions: " << tot[P_SPLIT] << " cut by an interface (" << tot[P_SPLITLOC] + << " nodes marching on here, " << tot[P_SPLITHAND] << " handed across)"; + if (tot[P_SPLITDROP] > 0) out << "\n Nodes lost in split pieces that came apart: " << tot[P_SPLITDROP]; + out << "\n"; + return out.str(); } diff --git a/SU2_CFD/src/drivers/CDriver.cpp b/SU2_CFD/src/drivers/CDriver.cpp index 5904ed9febb..fa16c685437 100644 --- a/SU2_CFD/src/drivers/CDriver.cpp +++ b/SU2_CFD/src/drivers/CDriver.cpp @@ -809,11 +809,14 @@ void CDriver::InitializeGeometryFVM(CConfig *config, CGeometry **&geometry) { /*--- Loop over all the new grid ---*/ + string pavingReports; + for (iMGlevel = 1; iMGlevel <= config->GetnMGLevels(); iMGlevel++) { /*--- Create main agglomeration structure ---*/ - geometry[iMGlevel] = new CMultiGridGeometry(geometry[iMGlevel-1], config, iMGlevel); + auto* coarse_grid = new CMultiGridGeometry(geometry[iMGlevel-1], config, iMGlevel); + geometry[iMGlevel] = coarse_grid; /*--- Protect against the situation that we were not able to complete the agglomeration for this level, i.e., there weren't enough points. @@ -824,6 +827,7 @@ void CDriver::InitializeGeometryFVM(CConfig *config, CGeometry **&geometry) { geometry[iMGlevel] = nullptr; break; } + pavingReports += coarse_grid->pavingReport; /*--- Compute points surrounding points. ---*/ @@ -850,6 +854,9 @@ void CDriver::InitializeGeometryFVM(CConfig *config, CGeometry **&geometry) { } + /*--- Held back so they do not interleave with the multigrid level table. ---*/ + if (rank == MASTER_NODE) cout << pavingReports; + if (config->GetWrt_MultiGrid()) geometry[MESH_0]->ColorMGLevels(config->GetnMGLevels(), geometry); /*--- For unsteady simulations, initialize the grid volumes diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 7e60d634c7a..2f034d40ae3 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -66,16 +66,7 @@ inline passivedouble ComputeLinSysResRMS(const CSolver* solver) { /*!\cond PRIVATE * Prolongate a coarse-grid field onto the fine grid via constant injection: every fine * child gets its parent's value. \c getCoarse returns the coarse-grid block of a point - * and \c setFine writes it to a fine-grid point, so the same loop serves both the FAS - * correction and the Full-MG solution handoff. - * - * The loop covers all coarse points, halos included. Halo coarse CVs own the fine halo points as - * children (CMultiGridGeometry sets Children_CV for received CVs), so injecting from them is what - * fills the fine-grid halo entries of the prolongated field. Restricting the loop to domain points - * leaves those entries at whatever the last solver update left there (zero, for LinSysRes), which - * is wrong for any operator that reads the prolongated field at neighbours across a partition - * boundary - the Jacobi smoother in SmoothProlongated_Correction does exactly that. The caller - * must therefore have synchronized the coarse-grid field being read before calling this. + * and \c setFine writes it to a fine-grid point. \endcond */ template void ProlongateField(CGeometry* geo_coarse, GetCoarse getCoarse, SetFine setFine) { From 3fc67361a20b2c82a99b406a97af31171a7d617c Mon Sep 17 00:00:00 2001 From: bigfooted Date: Mon, 7 Sep 2026 18:57:30 +0200 Subject: [PATCH 51/54] fix bug with cfl scaling with turbulence --- SU2_CFD/src/integration/CSingleGridIntegration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SU2_CFD/src/integration/CSingleGridIntegration.cpp b/SU2_CFD/src/integration/CSingleGridIntegration.cpp index 0f051fdc2b6..29c71b2ed36 100644 --- a/SU2_CFD/src/integration/CSingleGridIntegration.cpp +++ b/SU2_CFD/src/integration/CSingleGridIntegration.cpp @@ -49,8 +49,8 @@ void CSingleGridIntegration::SingleGrid_Iteration(CGeometry ****geometry, CSolve CGeometry* geometry_fine = geometry[iZone][iInst][FinestMesh]; CSolver** solvers_fine = solver_container[iZone][iInst][FinestMesh]; - if (RunTime_EqSystem == RUNTIME_TURB_SYS) { - /*--- CFL scaling of turbulence during the warmup phase if FMG. ---*/ + /*--- CFL scaling of turbulence during the warmpup phase if FMG. ---*/ + if ((RunTime_EqSystem == RUNTIME_TURB_SYS) && (FinestMesh != MESH_0)) { const su2double turbReduction = SU2_TYPE::GetValue(config[iZone]->GetCFLRedCoeff_Turb()); const su2double turbCFL = SU2_TYPE::GetValue(config[iZone]->GetCFL(FinestMesh)) * turbReduction; auto* turbSolver = solvers_fine[Solver_Position]; From 945186abbbe5de9f2d33da90fa9d8df94ae49f01 Mon Sep 17 00:00:00 2001 From: Nijso Date: Mon, 7 Sep 2026 19:38:47 +0200 Subject: [PATCH 52/54] Apply batched suggestions from code review Co-authored-by: Nijso --- .../include/geometry/CMultiGridGeometry.hpp | 2 +- Common/src/geometry/CMultiGridGeometry.cpp | 34 ++++--------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp index eee6038071e..e1ba5a80fde 100644 --- a/Common/include/geometry/CMultiGridGeometry.hpp +++ b/Common/include/geometry/CMultiGridGeometry.hpp @@ -87,7 +87,7 @@ class CMultiGridGeometry final : public CGeometry { su2double ComputeLocalCurvature(const CGeometry* fine_grid, unsigned long iPoint, unsigned short iMarker) const; /*! - * \brief Pave the domain with advancing fronts rising from the boundary patches. + * \brief Pave the domain with advancing fronts extruded from the boundary patches. * \param[in,out] Index_CoarseCV - Current coarse CV index, incremented as new coarse CVs are created. * \param[in] fine_grid - Fine grid geometry. * \param[in] config - Configuration. diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index d8eaaf9e98b..619ebc70ef1 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -98,19 +98,14 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } const auto endLineCV = Index_CoarseCV; - /*--- Points carrying a physical boundary condition. SEND_RECEIVE is not one: it only records that - * the point is mirrored on another rank. Used by the repair passes below to tell which coarse - * CVs touch a boundary, see cvOnBoundary. ---*/ + /*--- Points carrying a physical boundary condition. This does not include SEND_RECEIVE. ---*/ vector onPhysBoundary(fine_grid->GetnPoint(), 0); for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) onPhysBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } - /*--- Nodes where two different boundary conditions meet. The rule has to hold for every phase or - * the same node is treated one way by the paving and another here. In 2D the corner test below - * already refused them; in 3D a ridge of such nodes carries one identical marker PAIR all along - * it and would otherwise pair up with itself quite happily. ---*/ + /*--- Nodes where two different boundary conditions meet. ---*/ const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); /*--- STEP 1: The first step is the boundary agglomeration. ---*/ @@ -148,10 +143,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un marker_seed.push_back(iMarker); /*--- For a particular point in the fine grid we save all the physical markers that are in - that point. SEND_RECEIVE markers are deliberately not counted: including them would make - an ordinary wall point look like a ridge, and a wall/symmetry ridge look like a corner - (which the counter > 2 rule below then refuses to agglomerate at all), so a point would be - classified differently depending only on where the partition happens to cut. ---*/ + that point. ---*/ for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker(); jMarker++) { if (config->GetMarker_All_KindBC(jMarker) == SEND_RECEIVE) continue; @@ -205,12 +197,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un point merely split by a partition interface no longer reaches this branch: it counts one physical marker and is handled above as the valley point it is. ---*/ if (nDim == 2) agglomerate_seed = false; - /*--- In 3D, this is a ridge point (an edge feature where two surface markers meet). - Always allow it to attempt agglomeration here; SetBoundAgglomeration() enforces - the actual ridge-ridge rule downstream: it may only pair with a neighboring ridge - point that carries the identical physical marker pair. A mismatched marker pair - usually indicates a genuine sharp corner in the geometry and is correctly left - un-merged (falls through to the singleton leftover loop). ---*/ + /*--- In 3D, this is a ridge point (an edge where two surface markers meet). ---*/ if (nDim == 3) agglomerate_seed = true; /*--- Euler walls: check curvature-based agglomeration criterion for both markers ---*/ @@ -272,10 +259,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- Indirect neighbors only for 3D faces. The size test is needed on entry too: the sweep - * above leaves the CV exactly full when it hit the limit, and without this the first - * indirect candidate pushed it past the limit, after which the equality test never matched - * and the CV grew unbounded. ---*/ + /*--- Indirect neighbors only for 3D faces. ---*/ if ((nDim == 3) && (nChildren < maxAgglomSize)) { Suitable_Indirect_Neighbors.clear(); @@ -357,11 +341,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } /*--- STEP 2: Agglomerate the domain points. A seed grows one node at a time, taking the candidate - * that shares the most edges with the current members so the CV closes into a block rather than a - * star. Ties break on distance to the centroid measured in cells, using a local frame built from - * the seed's own edges, so a stretched cell does not pull the CV along the boundary layer. ---*/ - - /*--- Scratch shared by all seeds. The markers are cleared per CV, touching only what was used. ---*/ + * that shares the most edges with the current members. ---*/ vector inCV(fine_grid->GetnPoint(), 0); vector isCandidate(fine_grid->GetnPoint(), 0); vector members, candidates; @@ -369,7 +349,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- A local frame at the seed: up to nDim of its incident edges, as mutually orthogonal as possible, * each with its own length. An offset is resolved onto these and each component divided by that - * direction's spacing, so the measure turns with the mesh and stays correct on a curved boundary. ---*/ + * direction's spacing. ---*/ vector> frameDir, edgeDir; vector frameLen, edgeLen; vector edgeUsed; From 0d6321c2186976b99982736fb324e30ed4737e25 Mon Sep 17 00:00:00 2001 From: bigfooted Date: Mon, 7 Sep 2026 22:29:16 +0200 Subject: [PATCH 53/54] cleanup and re-arrange, update some cfg files --- Common/src/CConfig.cpp | 9 +- Common/src/geometry/CMultiGridGeometry.cpp | 667 ++++++++---------- .../integration/CMultiGridIntegration.hpp | 10 +- .../src/integration/CMultiGridIntegration.cpp | 8 +- TestCases/euler/channel/inv_channel_RK.cfg | 18 +- TestCases/euler/wedge/inv_wedge_HLLC.cfg | 8 +- TestCases/fixed_cl/naca0012/inv_NACA0012.cfg | 8 +- .../cylinder/cylinder_lowmach.cfg | 12 +- .../navierstokes/cylinder/lam_cylinder.cfg | 10 +- .../navierstokes/flatplate/lam_flatplate.cfg | 9 +- 10 files changed, 325 insertions(+), 434 deletions(-) diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index f4e5cc4d098..29161a74338 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -2061,11 +2061,13 @@ void CConfig::SetConfig_Options() { addDoubleOption("MG_SMOOTH_RES_THRESHOLD", MGOptions.MG_Smooth_Res_Threshold, 0.9); /*!\brief MG_SMOOTH_OUTPUT\n DESCRIPTION: Print compact per-cycle smoothing iteration summary. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_SMOOTH_OUTPUT", MGOptions.MG_Smooth_Output, false); - /*!\brief MG_SMOOTH_STAGNATION_TOL\n DESCRIPTION: Stop smoothing if current_rms >= previous_rms * this value. Values < 1.0 enable early exit on stagnation, 1.0 only exits on defect growth. DEFAULT: 0.99 \ingroup Config*/ + /*!\brief MG_SMOOTH_STAGNATION_TOL\n DESCRIPTION: Stop smoothing if current_rms >= previous_rms * this value. + * Values < 1.0 enable early exit on stagnation, 1.0 only exits on defect growth. DEFAULT: 0.99 \ingroup Config*/ addDoubleOption("MG_SMOOTH_STAGNATION_TOL", MGOptions.MG_Smooth_StagnationTol, 0.99); /*!\brief MG_SMOOTH_COEFF\n DESCRIPTION: Smoothing coefficient for the correction prolongation Jacobi smoother. DEFAULT: 1.25 \ingroup Config*/ addDoubleOption("MG_SMOOTH_COEFF", MGOptions.MG_Smooth_Coeff, 1.25); - /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 50 \ingroup Config*/ + /*!\brief MG_MIN_MESHSIZE\n DESCRIPTION: Minimum number of CVs on the coarsest multigrid level, checked per MPI rank + * (i.e. on the smallest partition). Levels that would produce fewer CVs on any rank are not created. DEFAULT: 500 \ingroup Config*/ addUnsignedLongOption("MG_MIN_MESHSIZE", MGOptions.MG_Min_MeshSize, 500); /*!\brief MG_IMPLICIT_LINES\n DESCRIPTION: Enable agglomeration along implicit lines from wall seeds. DEFAULT: NO \ingroup Config*/ addBoolOption("MG_IMPLICIT_LINES", MGOptions.MG_Implicit_Lines, false); @@ -2083,7 +2085,8 @@ void CConfig::SetConfig_Options() { /*!\brief MG_STARTUP_STAGNATION_ITER\n DESCRIPTION: Consecutive stalled iterations required before Full-MG promotes * on stagnation. 0 disables it, as MG_STARTUP_STAGNATION= 0 does. DEFAULT: 5 \ingroup Config*/ addUnsignedLongOption("MG_STARTUP_STAGNATION_ITER", MGOptions.MG_Startup_Stagnation_Iter, 5); - /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ + /*!\brief MG_CFL_SCALING\n DESCRIPTION: Per-level CFL scaling factors for coarse MG levels. Entry i is the ratio CFL(i+1)/CFL(i). + * If fewer values than nMGLevels are given, the last value is repeated. DEFAULT: 0.25 (i.e., 1/4 per level) \ingroup Config*/ addDoubleListOption("MG_CFL_SCALING", nMG_CflScaling_p, MG_CflScaling_p); /*!\par CONFIG_CATEGORY: Spatial Discretization \ingroup Config*/ diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index 619ebc70ef1..bd1877de9a4 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -1653,6 +1653,85 @@ bool VertexUnitNormal(const CGeometry* grid, unsigned short nDim, unsigned long return true; } +/*--- Are a and b neighbours in the fine grid? ---*/ +bool IsAdjacent(const CGeometry* grid, unsigned long a, unsigned long b) { + const auto& pts = grid->nodes->GetPoints(a); + return std::find(pts.begin(), pts.end(), b) != pts.end(); +} + +/*--- Is a footprint one connected patch? A set that falls into pieces is not the extrusion of + * anything, so an arriving footprint and a piece left by a split both have to pass this. ---*/ +bool IsConnectedLayer(const CGeometry* grid, const vector& layer) { + if (layer.size() < 2) return true; + vector seen(layer.size(), 0); + vector stk{0}; + seen[0] = 1; + size_t nSeen = 1; + while (!stk.empty()) { + const auto cur = stk.back(); + stk.pop_back(); + for (size_t k = 0; k < layer.size(); ++k) { + if (seen[k] || !IsAdjacent(grid, layer[cur], layer[k])) continue; + seen[k] = 1; + nSeen++; + stk.push_back(k); + } + } + return nSeen == layer.size(); +} + +/*--- Is the new layer topologically identical to the old? They are index-aligned, so phi maps + * old[k] to new[k], and the layer is valid when phi is an isomorphism of the induced subgraphs. ---*/ +bool LayerIsIsomorphic(const CGeometry* grid, const vector& oldL, const vector& newL) { + const auto n = oldL.size(); + if (newL.size() != n) return false; + + for (size_t k = 0; k < n; ++k) { + unsigned nOld = 0, nNew = 0; + for (size_t l = 0; l < n; ++l) { + nOld += IsAdjacent(grid, newL[k], oldL[l]); + nNew += IsAdjacent(grid, oldL[k], newL[l]); + } + /*--- Exactly one partner each way, and it has to be the one phi names. ---*/ + if ((nOld != 1) || (nNew != 1)) return false; + if (!IsAdjacent(grid, oldL[k], newL[k])) return false; + } + + for (size_t k = 0; k < n; ++k) + for (size_t l = k + 1; l < n; ++l) + if (IsAdjacent(grid, oldL[k], oldL[l]) != IsAdjacent(grid, newL[k], newL[l])) return false; + + return true; +} + +/*--- A boundary node ends a front only if the step runs INTO it, i.e. roughly along its normal. One + * running ALONG a boundary is a legitimate interior node of the stack. ---*/ +bool EntersBoundary(const CGeometry* grid, const CConfig* config, unsigned short nDim, unsigned long jPoint, + const su2double* stepDir, su2double cosBoundary) { + for (unsigned short iMarker = 0; iMarker < grid->GetnMarker(); iMarker++) { + if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; + su2double n[3] = {0.0}; /*--- nDim is at most 3. ---*/ + if (!VertexUnitNormal(grid, nDim, jPoint, iMarker, n)) continue; + if (fabs(GeometryToolbox::DotProduct(nDim, n, stepDir)) >= cosBoundary) return true; + } + return false; +} + +/*--- Fine layers the next coarse CV of this front holds: two, so the stack coarsens by the same + * factor along the marching direction as the footprint does across it. One only if a second + * layer would exceed the agglomeration size limit. ---*/ +unsigned long BlockFor(short int maxAgglomSize, const vector& layer) { + return (layer.size() * 2 > static_cast(maxAgglomSize)) ? 1 : 2; +} + +/*--- Rank-independent name for a set of nodes: the smallest global index in it, +1 so that 0 is + * free to mean "nothing". A footprint is claimed by one front and no other, so it is unique. ---*/ +unsigned long TagOfSet(const CGeometry* grid, const vector& set) { + unsigned long t = std::numeric_limits::max(); + for (auto p : set) t = std::min(t, grid->nodes->GetGlobalIndex(p)); + return t + 1; +} + } // namespace CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeometry* fine_grid, const CConfig* config, @@ -1781,29 +1860,15 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront const auto nSeeds = seeds.node.size(); const unsigned long max_group = (nDim == 2) ? 2 : 4; - /*--- Marker signature of each seed, as a bitmask over the physical markers. Seeds may only be - * matched when these agree, so a patch never straddles a change of boundary condition - the rule - * ordinary agglomeration uses for ridges and valleys. ---*/ + /*--- Physical markers each seed lies on, ascending. Seeds may only be matched when these agree, so + * a patch never straddles a change of boundary condition. ---*/ const auto nMarkerFine = fine_grid->GetnMarker(); - vector physBit(nMarkerFine, -1); - unsigned nPhys = 0; - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) - if (config->GetMarker_All_KindBC(iMarker) != SEND_RECEIVE) physBit[iMarker] = static_cast(nPhys++); - - const unsigned nWords = std::max(1u, (nPhys + 63u) / 64u); - vector sig(nSeeds * nWords, 0); + vector> sig(nSeeds); for (unsigned long si = 0; si < nSeeds; ++si) - for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { - if (physBit[iMarker] < 0) continue; - if (fine_grid->nodes->GetVertex(seeds.node[si], iMarker) == -1) continue; - const auto b = static_cast(physBit[iMarker]); - sig[si * nWords + b / 64] |= (uint64_t(1) << (b % 64)); - } - auto sameSig = [&](unsigned long sa, unsigned long sb) { - for (unsigned w = 0; w < nWords; ++w) - if (sig[sa * nWords + w] != sig[sb * nWords + w]) return false; - return true; - }; + for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) + if ((config->GetMarker_All_KindBC(iMarker) != SEND_RECEIVE) && + (fine_grid->nodes->GetVertex(seeds.node[si], iMarker) != -1)) + sig[si].push_back(iMarker); /*--- Seed-to-seed adjacency, inherited from the boundary nodes' mesh connectivity. ---*/ vector seedOfNode(fine_grid->GetnPoint(), -1); @@ -1846,7 +1911,7 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront }; vector merges; - vector> shared; + vector touched, nShared(nSeeds, 0); vector gkey; vector consumed; @@ -1866,25 +1931,20 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront * rising from it is one node wide. Both sides of a merge are tested: skipping only the mixed * group would still let an ordinary group reach out and take it. ---*/ if (mixedBC[seeds.node[groups[g].front()]]) continue; - shared.clear(); + touched.clear(); for (auto si : groups[g]) for (auto sj : adj[si]) { const auto h = groupOf[sj]; if (h <= g) continue; if (mixedBC[seeds.node[groups[h].front()]]) continue; if (groups[g].size() + groups[h].size() > max_group) continue; - if (!sameSig(groups[h].front(), groups[g].front())) continue; - - bool seen = false; - for (auto& t : shared) - if (t.first == h) { - t.second++; - seen = true; - break; - } - if (!seen) shared.emplace_back(h, 1); + if (sig[groups[h].front()] != sig[groups[g].front()]) continue; + if (nShared[h]++ == 0) touched.push_back(h); } - for (const auto& t : shared) merges.push_back({g, t.first, t.second, gkey[g], gkey[t.first]}); + for (auto h : touched) { + merges.push_back({g, h, nShared[h], gkey[g], gkey[h]}); + nShared[h] = 0; + } } /*--- Best merges first over all groups at once. Sweeping groups in index order lets one with no @@ -1921,12 +1981,9 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, unsigned short iMesh) { - /*--- Paving by advancing fronts. Each boundary patch from PHASE 1 rises into the domain keeping its - * footprint: every front node picks a successor and the front advances only if all succeed, so the - * layers of a stack are congruent and a coarse CV never spans two patches. A front stops at a - * boundary or when the next layer is not isomorphic to the current one. Each coarse CV is the - * footprint two layers deep. The multigrid queue is not touched here; the sync loop after boundary - * agglomeration already removes every point marked agglomerated. ---*/ + /*--- Paving by advancing fronts. Each boundary patch rises into the domain keeping its footprint, so + * a coarse CV never spans two patches. A front stops at a boundary or when the next layer is not + * isomorphic to the current one. Each coarse CV is the footprint two layers deep. ---*/ const auto starting_Index_CoarseCV = Index_CoarseCV; const auto nPointFine = fine_grid->GetnPoint(); const auto nMarkerFine = fine_grid->GetnMarker(); @@ -1960,20 +2017,6 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC onPhysicalBoundary[fine_grid->vertex[iMarker][iVertex]->GetNode()] = 1; } - /*--- A boundary node ends a front only if the step runs INTO that boundary, i.e. roughly parallel to - * its normal. A node running ALONG a boundary, such as a column on a spanwise symmetry plane, is a - * legitimate interior node of the stack; onPhysicalBoundary records marker membership only and - * cannot tell the two apart. ---*/ - auto entersBoundary = [&](unsigned long jPoint, const su2double* stepDir) { - for (unsigned short iMarker = 0; iMarker < nMarkerFine; iMarker++) { - if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; - su2double n[MAXNDIM] = {0.0}; - if (!VertexUnitNormal(fine_grid, nDim, jPoint, iMarker, n)) continue; - if (fabs(GeometryToolbox::DotProduct(nDim, n, stepDir)) >= cos_boundary) return true; - } - return false; - }; - /*================================================================================================== * PHASE 2 - advance every front, one layer per round. *================================================================================================*/ @@ -1988,62 +2031,89 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC su2double dir[MAXNDIM]; /*!< Unit step direction, reused to update the front's direction. */ }; - /*--- Fronts are not a fixed set: one handed over from a neighbouring rank is appended while the - * rounds are running, so every per-front array grows and the loops below are bounded by - * front.size() rather than by the number of patches. ---*/ - vector> front, pending, handTo; - vector> dirNow; - vector alive; - vector depth, nBlock, pendingLayers, tag; - vector> prop; - - /*--- A name for a front that means the same thing on every rank, so a stack handed across a - * partition can be recognised on the far side and so two ranks reaching for the same node can be - * separated the same way by both. The smallest global point index of the patch it grew from is - * unique, since a seed belongs to exactly one patch; the +1 leaves 0 free to mean "nothing". ---*/ + /*--- One advancing front. Fronts are not a fixed set: one handed over from a neighbouring rank is + * appended while the rounds are running, so the loops below are bounded by fronts.size(). ---*/ + struct CFront { + vector nodes; /*!< \brief Current footprint. */ + vector pending; /*!< \brief Nodes buffered for the coarse CV being built. */ + vector handTo; /*!< \brief Halo nodes the stack should continue onto. */ + vector prop; /*!< \brief This round's proposed successors. */ + std::array dir{}; /*!< \brief Marching direction. */ + unsigned long tag = 0; /*!< \brief Rank-independent name, see TagOfSet. */ + unsigned long handTag = 0; /*!< \brief Name the handed-over piece travels under. After a + * split this is NOT tag: the pieces are separate stacks. */ + unsigned long depth = 0; /*!< \brief Layers laid. */ + unsigned long nBlock = 0; /*!< \brief Fine layers the next coarse CV holds. */ + unsigned long pendingLayers = 0; /*!< \brief Layers currently buffered. */ + char alive = 1; + char failed = 0; + char keepLocal = 0; /*!< \brief Handed over only part of its footprint, so it marches on here. */ + }; + vector fronts; + auto addFront = [&](const vector& layer, const std::array& dir, unsigned long frontTag, unsigned long block) { - front.push_back(layer); - pending.push_back(layer); - handTo.emplace_back(); - dirNow.push_back(dir); - alive.push_back(1); - depth.push_back(0); - nBlock.push_back(block); - pendingLayers.push_back(1); - tag.push_back(frontTag); - prop.emplace_back(); - return front.size() - 1; + CFront F; + F.nodes = F.pending = layer; + F.dir = dir; + F.tag = frontTag; + F.nBlock = block; + F.pendingLayers = 1; + fronts.push_back(std::move(F)); + return fronts.size() - 1; }; vector claimed(nPointFine, 0); - /*--- Confirmed owner of a claimed node, -1 while free. Only ever written when a layer is accepted, - * so a bid that is still being contested never appears here. ---*/ - - vector failed; - - /*--- Set when a front hands only PART of its footprint over and goes on marching here with what is - * left of it, so the retirement pass at the end of the round knows not to kill it. ---*/ - vector keepLocal; - /*--- The name the handed-over piece travels under. After a split this is NOT the name of the front - * it came from: the two pieces are separate stacks from here on, and giving them one name would - * let the far side group a piece of this stack with a piece of another one. ---*/ - vector handTag; - - /*--- A name for a set of nodes that both ranks sharing them would compute identically. The nodes of - * a footprint are claimed by one front and by no other, so the smallest global index in it is a - * unique name for that front; the +1 leaves 0 free to mean "nothing". ---*/ - auto tagOfSet = [&](const vector& set) { - unsigned long t = std::numeric_limits::max(); - for (auto p : set) t = std::min(t, fine_grid->nodes->GetGlobalIndex(p)); - return t + 1; + + /*--- Best free neighbour of n to step onto, ranked by alignment with dir. Local and halo candidates + * are ranked separately: a halo node cannot be claimed here, but it is where the stack would go + * next, so it is what gets handed over. ---*/ + struct CCandidate { + unsigned long node = std::numeric_limits::max(); + unsigned long halo = std::numeric_limits::max(); + su2double dot = -2.0, len = 0.0, dir[MAXNDIM] = {0.0}; }; - /*--- The bid table. Only the index is kept per mesh point, and the bids themselves live in a - * compact vector holding one entry per candidate actually bid on this round - a few per front, - * against one entry per point in the mesh. Storing a whole CStep per point instead costs about - * sixty bytes times nPoint, which on the meshes this code is meant for is hundreds of megabytes - * of table that is empty almost everywhere. ---*/ + auto bestSuccessor = [&](unsigned long n, const su2double* marchDir) { + CCandidate c; + su2double haloDot = -2.0; + + for (auto jPoint : fine_grid->nodes->GetPoints(n)) { + su2double vec[MAXNDIM] = {0.0}; + GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(n), vec); + const su2double len = GeometryToolbox::Norm(nDim, vec); + if (len <= 0.0) continue; + for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; + + /*--- The direction only RANKS candidates; a front is never stopped for turning. ---*/ + const su2double dot = GeometryToolbox::DotProduct(nDim, vec, marchDir); + const bool admissible = + !(onPhysicalBoundary[jPoint] && EntersBoundary(fine_grid, config, nDim, jPoint, vec, cos_boundary)) && + GeometricalCheck(jPoint, fine_grid, config); + + /*--- Halo parents are assigned by the owning rank through the MPI relay, so claiming one here + * would fight that assignment. Admissibility is still checked, as the owner would. ---*/ + if (!fine_grid->nodes->GetDomain(jPoint)) { + if (dot > haloDot && admissible) { + haloDot = dot; + c.halo = jPoint; + } + continue; + } + if (fine_grid->nodes->GetAgglomerate(jPoint) || claimed[jPoint] || !admissible) continue; + + if (dot > c.dot) { + c.dot = dot; + c.node = jPoint; + c.len = len; + for (unsigned short d = 0; d < nDim; ++d) c.dir[d] = vec[d]; + } + } + return c; + }; + + /*--- Bid table: only an index per mesh point, the bids themselves in a compact vector. A whole + * CStep per point would be hundreds of megabytes that is empty almost everywhere. ---*/ constexpr unsigned NOBID = std::numeric_limits::max(); vector bidIdx(nPointFine, NOBID); vector bids; @@ -2052,22 +2122,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC /*--- Scratch for the layer under construction, hoisted so a front does not allocate per layer. ---*/ vector newLayer; - /*--- Paving diagnostics, summed over all ranks in one reduction at the end. ---*/ - enum { - P_STACKS, - P_LAYERS, - P_COVERED, - P_SEMICV, - P_FULLCV, - P_HANDOUT, - P_HANDIN, - P_SPLIT, - P_SPLITLOC, - P_SPLITHAND, - P_SPLITDROP, - P_HIST, /*!< \brief Start of nine patch-size bins. */ - P_COUNT = P_HIST + 9 - }; + /*--- Summed over all ranks for the one-line report at the end. ---*/ + enum { P_STACKS, P_LAYERS, P_COVERED, P_COUNT }; unsigned long ct[P_COUNT] = {0}; /*--- One footprint node arriving from a neighbouring rank, to be regrouped by tag. ---*/ @@ -2093,85 +2149,27 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } } - auto markFail = [&](unsigned long f) { failed[f] = 1; }; + auto markFail = [&](unsigned long f) { fronts[f].failed = 1; }; /*--- How many fine layers the next coarse CV of this front holds: two, so the stack coarsens by the * same factor along the marching direction as the footprint does across it. One only if a second * layer would exceed the agglomeration size limit. ---*/ - auto blockFor = [&](const vector& layer) -> unsigned long { - return (layer.size() * 2 > static_cast(maxAgglomSize)) ? 1 : 2; - }; - /*--- Turn everything buffered for this front into one coarse control volume. ---*/ auto emit = [&](unsigned long f) { - if (pending[f].empty()) return; - for (unsigned long c = 0; c < pending[f].size(); ++c) { - const auto p = pending[f][c]; + if (fronts[f].pending.empty()) return; + for (unsigned long c = 0; c < fronts[f].pending.size(); ++c) { + const auto p = fronts[f].pending[c]; fine_grid->nodes->SetParent_CV(p, Index_CoarseCV); nodes->SetChildren_CV(Index_CoarseCV, c, p); if (fine_grid->nodes->GetAgglomerate_Indirect(p)) nodes->SetAgglomerate_Indirect(Index_CoarseCV, true); } - nodes->SetnChildren_CV(Index_CoarseCV, static_cast(pending[f].size())); + nodes->SetnChildren_CV(Index_CoarseCV, static_cast(fronts[f].pending.size())); Index_CoarseCV++; - ct[P_COVERED] += pending[f].size(); - ct[(pendingLayers[f] == 1) ? P_SEMICV : P_FULLCV]++; - - pending[f].clear(); - pendingLayers[f] = 0; - nBlock[f] = blockFor(front[f]); - }; - - auto isAdjacent = [&](unsigned long a, unsigned long b) { - const auto& pts = fine_grid->nodes->GetPoints(a); - return std::find(pts.begin(), pts.end(), b) != pts.end(); - }; - - /*--- Is a footprint one connected patch? A set that falls into pieces is not the extrusion of - * anything, so a footprint arriving from a neighbour and a piece left behind by a split both - * have to pass this before they are allowed to carry a stack. ---*/ - auto isConnectedLayer = [&](const vector& layer) { - if (layer.size() < 2) return true; - vector seen(layer.size(), 0); - vector stk{0}; - seen[0] = 1; - size_t nSeen = 1; - while (!stk.empty()) { - const auto cur = stk.back(); - stk.pop_back(); - for (size_t k = 0; k < layer.size(); ++k) { - if (seen[k] || !isAdjacent(layer[cur], layer[k])) continue; - seen[k] = 1; - nSeen++; - stk.push_back(k); - } - } - return nSeen == layer.size(); - }; - - /*--- Is the layer about to be laid topologically identical to the one below? "old" and "new" are - * index-aligned, so phi maps old[k] to new[k]. The layer is valid when phi is an isomorphism of - * the two induced subgraphs: each new cell adjacent to exactly its own preimage, each old cell to - * exactly its own image, and old[k]-old[l] an edge if and only if new[k]-new[l] is. ---*/ - auto layerIsIsomorphic = [&](const vector& oldL, const vector& newL) { - const auto n = oldL.size(); - if (newL.size() != n) return false; - - for (size_t k = 0; k < n; ++k) { - unsigned nOld = 0, nNew = 0; - for (size_t l = 0; l < n; ++l) { - nOld += isAdjacent(newL[k], oldL[l]); - nNew += isAdjacent(oldL[k], newL[l]); - } - /*--- Exactly one partner each way, and it has to be the one phi names. ---*/ - if ((nOld != 1) || (nNew != 1)) return false; - if (!isAdjacent(oldL[k], newL[k])) return false; - } - - for (size_t k = 0; k < n; ++k) - for (size_t l = k + 1; l < n; ++l) - if (isAdjacent(oldL[k], oldL[l]) != isAdjacent(newL[k], newL[l])) return false; + ct[P_COVERED] += fronts[f].pending.size(); - return true; + fronts[f].pending.clear(); + fronts[f].pendingLayers = 0; + fronts[f].nBlock = BlockFor(maxAgglomSize, fronts[f].nodes); }; /*--- The boundary layer of every front. Claiming it before ordinary boundary agglomeration runs is @@ -2200,35 +2198,28 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (nrm <= 0.0) continue; for (unsigned short d = 0; d < nDim; ++d) n0[d] /= nrm; - /*--- The boundary layer becomes a coarse CV on its own: a boundary node is never merged with an - * interior one, so the first advance of every front is a single layer. Only the first - emit() - * then asks blockFor again for what is by then an ordinary interior layer, and the rest of the - * stack rises two nodes at a time. This is also what isolates a junction node without needing - * a rule of its own: such a node is a patch of one, so its first CV holds it and nothing else. ---*/ + /*--- The boundary layer is a coarse CV on its own: a boundary node is never merged with an + * interior one, so only the first advance is a single layer. This also isolates a junction + * node without a rule of its own, since such a node is a patch of one. ---*/ const auto f = addFront(layer0, n0, frontTag + 1, 1); for (auto p : layer0) { claimed[p] = 1; } - ct[P_HIST + std::min(layer0.size(), 8)]++; ct[P_STACKS]++; ct[P_LAYERS]++; emit(f); } for (unsigned long layer = 1;; ++layer) { - /*--- Whether ANY rank still has a live front, not just this one. Every rank has to run the same - * number of rounds because each round ends in a handover exchange that they all take part in: - * a rank whose own fronts are long finished may still be about to receive a stack from a - * neighbour, and a rank that dropped out of the loop early would hang the ones that did not. ---*/ + /*--- Every rank runs the same number of rounds: each ends in a collective handover exchange, so + * one dropping out early would hang the others. ---*/ int aliveLocal = 0; - for (unsigned long f = 0; f < front.size(); ++f) aliveLocal |= alive[f]; + for (unsigned long f = 0; f < fronts.size(); ++f) aliveLocal |= fronts[f].alive; int aliveGlobal = 0; SU2_MPI::Allreduce(&aliveLocal, &aliveGlobal, 1, MPI_INT, MPI_MAX, SU2_MPI::GetComm()); if (aliveGlobal == 0) break; - failed.assign(front.size(), 0); - keepLocal.assign(front.size(), 0); - handTag.assign(front.size(), 0); + for (auto& F : fronts) F.failed = F.keepLocal = F.handTag = 0; for (const auto& b : bids) bidIdx[b.node] = NOBID; bids.clear(); bidOwner.clear(); @@ -2236,118 +2227,62 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC /*--- (a) Every alive front proposes a successor for each of its nodes. A front that cannot fill a * whole layer proposes NOTHING: it is retiring this round anyway, and letting its partial bids * stand would let a dying front displace a healthy one out of nodes it can still use. ---*/ - for (unsigned long f = 0; f < front.size(); ++f) { - if (!alive[f]) continue; - prop[f].clear(); - handTo[f].clear(); - - for (auto n : front[f]) { - auto best = NO_POINT; - su2double best_dot = -2.0, best_len = 0.0, best_dir[MAXNDIM] = {0.0}; - /*--- The best step onto a node this rank does NOT own, kept separately. It cannot be claimed - * here, but it is where the stack would go next, so it is what gets handed over. ---*/ - auto bestHalo = NO_POINT; - su2double bestHalo_dot = -2.0; - - for (auto iNeigh = 0u; iNeigh < fine_grid->nodes->GetnPoint(n); ++iNeigh) { - const auto jPoint = fine_grid->nodes->GetPoint(n, iNeigh); - - su2double vec[MAXNDIM] = {0.0}; - GeometryToolbox::Distance(nDim, fine_grid->nodes->GetCoord(jPoint), fine_grid->nodes->GetCoord(n), vec); - const su2double len = GeometryToolbox::Norm(nDim, vec); - if (len <= 0.0) continue; - for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - - /*--- The marching direction RANKS the candidates and nothing more: whichever free neighbour - * lies most nearly ahead is the one proposed. There is no cone, so a front is never - * stopped for turning - only for running out of mesh to extrude into. ---*/ - const su2double dot = GeometryToolbox::DotProduct(nDim, vec, dirNow[f].data()); - - /*--- Halo nodes stay out: their parent is dictated by the rank that owns them and arrives - * through the MPI relay, so a front claiming one would fight that assignment. This is - * what a front hits when it reaches a partition interface, and it needs a reason of its - * own, rather than being skipped silently and leaving some other candidate to explain a - * stop that was really the partitioning. ---*/ - if (!fine_grid->nodes->GetDomain(jPoint)) { - /*--- Held as a handover candidate, subject to the same admissibility the owner would - * apply anyway; whether it is still free is the owner's to decide. ---*/ - if (dot > bestHalo_dot && !(onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) && - GeometricalCheck(jPoint, fine_grid, config)) { - bestHalo_dot = dot; - bestHalo = jPoint; - } - continue; - } - /*--- Taken by an earlier phase or by another front's coarse CV. ---*/ - if (fine_grid->nodes->GetAgglomerate(jPoint)) continue; - /*--- Already bid for this round. ---*/ - if (claimed[jPoint]) continue; - /*--- The step runs into a boundary, or the node would make an unusable coarse cell. ---*/ - if (onPhysicalBoundary[jPoint] && entersBoundary(jPoint, vec)) continue; - if (!GeometricalCheck(jPoint, fine_grid, config)) continue; - - if (dot > best_dot) { - best_dot = dot; - best = jPoint; - best_len = len; - for (unsigned short d = 0; d < nDim; ++d) best_dir[d] = vec[d]; - } - } - - if ((best == NO_POINT) && (bestHalo != NO_POINT)) { - /*--- Nowhere left on this rank, but the stack does continue - just on someone else's side - * of the interface. Remember where, and let the classification below decide whether the - * whole layer goes over. ---*/ - handTo[f].push_back(bestHalo); + for (unsigned long f = 0; f < fronts.size(); ++f) { + if (!fronts[f].alive) continue; + fronts[f].prop.clear(); + fronts[f].handTo.clear(); + + for (auto n : fronts[f].nodes) { + const auto c = bestSuccessor(n, fronts[f].dir.data()); + + /*--- Nothing free here but the stack continues across the interface; the split test below + * decides whether the whole layer goes over. ---*/ + if ((c.node == NO_POINT) && (c.halo != NO_POINT)) { + fronts[f].handTo.push_back(c.halo); continue; } - - /*--- No admissible successor: a boundary, a partition, another front, or unusable mesh. ---*/ - if (best == NO_POINT) { + /*--- No successor at all: a boundary, a partition, another front, or unusable mesh. ---*/ + if (c.node == NO_POINT) { markFail(f); break; } - CStep s{best, n, fine_grid->nodes->GetGlobalIndex(n), best_dot, best_len, {}}; - for (unsigned short d = 0; d < nDim; ++d) s.dir[d] = best_dir[d]; - prop[f].push_back(s); + CStep s{c.node, n, fine_grid->nodes->GetGlobalIndex(n), c.dot, c.len, {}}; + for (unsigned short d = 0; d < nDim; ++d) s.dir[d] = c.dir[d]; + fronts[f].prop.push_back(s); } /*--- An interface can cut a footprint. If all of it crosses, the stack is handed over intact * and this front is finished. If only part crosses, the footprint is SPLIT: the piece whose * successors are local marches on here, the rest is handed across, and both are renamed. ---*/ - if (failed[f]) { - prop[f].clear(); - handTo[f].clear(); - } else if (!handTo[f].empty()) { - /*--- prop[f] is built in the order of front[f], so this is the piece that stays, in the same + if (fronts[f].failed) { + fronts[f].prop.clear(); + fronts[f].handTo.clear(); + } else if (!fronts[f].handTo.empty()) { + /*--- fronts[f].prop is built in the order of fronts[f].nodes, so this is the piece that stays, in the same * order, and phi still runs index for index between it and the layer it proposes. ---*/ vector narrow; - for (const auto& s : prop[f]) narrow.push_back(s.from); + for (const auto& s : fronts[f].prop) narrow.push_back(s.from); /*--- A cut can leave the local piece in two disconnected halves - a square footprint cut * diagonally does exactly that - and that is not a layer. Drop it and hand over the rest; * the stack still survives on the far side instead of ending here. ---*/ - if (!narrow.empty() && !isConnectedLayer(narrow)) { - ct[P_SPLITDROP] += narrow.size(); + if (!narrow.empty() && !IsConnectedLayer(fine_grid, narrow)) { narrow.clear(); } - handTag[f] = tagOfSet(handTo[f]); + fronts[f].handTag = TagOfSet(fine_grid, fronts[f].handTo); if (narrow.empty()) { - prop[f].clear(); + fronts[f].prop.clear(); } else { - ct[P_SPLIT]++; - ct[P_SPLITLOC] += narrow.size(); - ct[P_SPLITHAND] += handTo[f].size(); /*--- Close the coarse CV that is open on the WIDE footprint before narrowing, so that no CV * ever ends up holding two layers of different shape. ---*/ - front[f] = narrow; + fronts[f].nodes = narrow; emit(f); - nBlock[f] = blockFor(front[f]); - tag[f] = tagOfSet(front[f]); - keepLocal[f] = 1; + fronts[f].nBlock = BlockFor(maxAgglomSize, fronts[f].nodes); + fronts[f].tag = TagOfSet(fine_grid, fronts[f].nodes); + fronts[f].keepLocal = 1; } } } @@ -2361,14 +2296,12 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC return a.key < b.key; }; - for (unsigned long f = 0; f < front.size(); ++f) { - if (!alive[f] || failed[f]) continue; - for (const auto& s : prop[f]) { - /*--- A front that has lost a bid is retiring and must not place the rest of its layer and - * displace a healthy front. What it placed before losing stays, which can still cost another - * front a candidate; the residual is conservative and retires a front near a seam one layer - * early, and the seam goes to ordinary agglomeration either way. ---*/ - if (failed[f]) break; + for (unsigned long f = 0; f < fronts.size(); ++f) { + if (!fronts[f].alive || fronts[f].failed) continue; + for (const auto& s : fronts[f].prop) { + /*--- A front that has lost a bid is retiring and must not displace a healthy one with the + * rest of its layer. What it placed before losing stays, which is conservative. ---*/ + if (fronts[f].failed) break; if (bidIdx[s.node] == NOBID) { bidIdx[s.node] = static_cast(bids.size()); @@ -2389,11 +2322,9 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC markFail(f); } - /*--- A head-on meeting stops BOTH fronts. Letting the winner carry on through the seam would - * push its stack into territory the other front had every right to, and the asymmetry - * shows up in the coarse grid as one stack overshooting the other. A glancing contact - * (directions not opposed) is not a meeting and only costs the loser. ---*/ - if ((g != f) && (GeometryToolbox::DotProduct(nDim, dirNow[f].data(), dirNow[g].data()) < 0.0)) { + /*--- A head-on meeting stops BOTH fronts, or the winner overshoots into the other's + * territory. A glancing contact is not a meeting and only costs the loser. ---*/ + if ((g != f) && (GeometryToolbox::DotProduct(nDim, fronts[f].dir.data(), fronts[g].dir.data()) < 0.0)) { markFail(f); markFail(g); } @@ -2403,58 +2334,52 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC /*--- (c) All-or-nothing acceptance: a front takes the whole layer or none of it and retires. A * bid only becomes a claim here, so a retiring front never has to give anything back and the * nodes it was reaching for stay available to ordinary agglomeration. ---*/ - for (unsigned long f = 0; f < front.size(); ++f) { - if (!alive[f]) continue; + for (unsigned long f = 0; f < fronts.size(); ++f) { + if (!fronts[f].alive) continue; newLayer.clear(); - if (!failed[f]) { - /*--- Built in the order prop[f] was, which is the order of front[f], so newLayer[k] is the - * successor proposed by front[f][k] and the two vectors carry phi between them. ---*/ - for (const auto& s : prop[f]) { + if (!fronts[f].failed) { + /*--- Built in proposal order, so newLayer[k] is the successor of nodes[k] and the two carry phi. ---*/ + for (const auto& s : fronts[f].prop) { const auto k = bidIdx[s.node]; if ((k != NOBID) && (bidOwner[k] == f)) newLayer.push_back(s.node); } /*--- Every bid of a front that was not marked failed must have been granted. ---*/ - if ((newLayer.size() != front[f].size()) || !layerIsIsomorphic(front[f], newLayer)) markFail(f); + if ((newLayer.size() != fronts[f].nodes.size()) || !LayerIsIsomorphic(fine_grid, fronts[f].nodes, newLayer)) + markFail(f); } - if (failed[f]) { - /*--- Nothing to give back: a bid only becomes a claim on acceptance below. ---*/ - alive[f] = 0; - /*--- One layer short of a full block at the top: take what is buffered as its own coarse CV - * rather than dropping it back to ordinary agglomeration. ---*/ + if (fronts[f].failed) { + /*--- Nothing to give back: a bid only becomes a claim on acceptance. ---*/ + fronts[f].alive = 0; emit(f); continue; } - /*--- (d) Hand stacks across partition interfaces. A front that runs into the halo cannot go on - * here, so its footprint is sent to the owning rank, which picks the stack up and carries on. - * What crosses is the footprint, not a coarse CV, so both halves keep the same shape. It travels - * the reverse of the usual halo direction: packed against the RECEIVE marker and sent to the - * rank that marker receives from. ---*/ + /*--- Turn the marching direction towards the mean of the steps just taken. ---*/ su2double mean[MAXNDIM] = {0.0}; - for (const auto& s : prop[f]) + for (const auto& s : fronts[f].prop) for (unsigned short d = 0; d < nDim; ++d) mean[d] += s.dir[d]; const su2double meanNrm = GeometryToolbox::Norm(nDim, mean); if (meanNrm > 0.0) { su2double blended[MAXNDIM] = {0.0}; for (unsigned short d = 0; d < nDim; ++d) - blended[d] = (1.0 - DIR_BLEND) * dirNow[f][d] + DIR_BLEND * mean[d] / meanNrm; + blended[d] = (1.0 - DIR_BLEND) * fronts[f].dir[d] + DIR_BLEND * mean[d] / meanNrm; const su2double bNrm = GeometryToolbox::Norm(nDim, blended); if (bNrm > 0.0) - for (unsigned short d = 0; d < nDim; ++d) dirNow[f][d] = blended[d] / bNrm; + for (unsigned short d = 0; d < nDim; ++d) fronts[f].dir[d] = blended[d] / bNrm; } for (auto p : newLayer) { claimed[p] = 1; } - front[f] = std::move(newLayer); - depth[f]++; + fronts[f].nodes = std::move(newLayer); + fronts[f].depth++; ct[P_LAYERS]++; - pending[f].insert(pending[f].end(), front[f].begin(), front[f].end()); - pendingLayers[f]++; - if (pendingLayers[f] >= nBlock[f]) emit(f); + fronts[f].pending.insert(fronts[f].pending.end(), fronts[f].nodes.begin(), fronts[f].nodes.end()); + fronts[f].pendingLayers++; + if (fronts[f].pendingLayers >= fronts[f].nBlock) emit(f); } /*--- (d) Hand stacks across partition interfaces. A front that runs into the halo cannot go on @@ -2472,22 +2397,20 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC const auto nVertexS = fine_grid->nVertex[MarkerS]; const auto nVertexR = fine_grid->nVertex[MarkerR]; - /*--- Packed against the halo vertices, i.e. what this rank wants the neighbour to continue. ---*/ + /*--- Packed against the halo vertices, i.e. what this rank wants the neighbour to continue. + * Tag and direction go separately: the AD MPI wrapper has no byte type to send a struct. ---*/ vector tagOut(nVertexR, 0), tagIn(nVertexS, 0); vector dirOut(nVertexR * nDim, 0.0), dirIn(nVertexS * nDim, 0.0); - for (unsigned long f = 0; f < front.size(); ++f) { - if (handTo[f].empty()) continue; - for (auto p : handTo[f]) { + for (auto& F : fronts) + for (auto p : F.handTo) { if (haloMarker[p] != static_cast(MarkerR)) continue; const auto v = haloVertex[p]; - /*--- Two fronts of this rank reaching for the same node: the lower tag takes it, which is - * a decision both ranks would reach the same way. ---*/ - if ((tagOut[v] != 0) && (tagOut[v] <= handTag[f])) continue; - tagOut[v] = handTag[f]; - for (unsigned short d = 0; d < nDim; ++d) dirOut[v * nDim + d] = dirNow[f][d]; + /*--- Two fronts reaching for one node: the lower tag takes it, the same way on both ranks. ---*/ + if ((tagOut[v] != 0) && (tagOut[v] <= F.handTag)) continue; + tagOut[v] = F.handTag; + for (unsigned short d = 0; d < nDim; ++d) dirOut[v * nDim + d] = F.dir[d]; } - } SU2_MPI::Sendrecv(tagOut.data(), nVertexR, MPI_UNSIGNED_LONG, receive_from, 2, tagIn.data(), nVertexS, MPI_UNSIGNED_LONG, send_to, 2, SU2_MPI::GetComm(), MPI_STATUS_IGNORE); @@ -2496,29 +2419,23 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC for (auto iVertex = 0ul; iVertex < nVertexS; iVertex++) { if (tagIn[iVertex] == 0) continue; - CInherited h; - h.tag = tagIn[iVertex]; - h.node = fine_grid->vertex[MarkerS][iVertex]->GetNode(); - for (unsigned short d = 0; d < nDim; ++d) h.dir[d] = dirIn[iVertex * nDim + d]; - inherited.push_back(h); + inherited.push_back({tagIn[iVertex], fine_grid->vertex[MarkerS][iVertex]->GetNode(), {}}); + for (unsigned short d = 0; d < nDim; ++d) inherited.back().dir[d] = dirIn[iVertex * nDim + d]; } } /*--- A front that handed its WHOLE footprint over is finished here; the neighbour owns the rest * of the stack. One that handed over only a piece keeps marching on what was left of it. ---*/ - for (unsigned long f = 0; f < front.size(); ++f) { - if (handTo[f].empty()) continue; - handTo[f].clear(); - ct[P_HANDOUT]++; - if (keepLocal[f]) continue; - alive[f] = 0; + for (unsigned long f = 0; f < fronts.size(); ++f) { + if (fronts[f].handTo.empty()) continue; + fronts[f].handTo.clear(); + if (fronts[f].keepLocal) continue; + fronts[f].alive = 0; emit(f); } - /*--- Adopt what the neighbours sent. Tags are processed in ascending order so that two ranks - * handing stacks onto overlapping nodes are separated the same way whatever order the messages - * happened to arrive in. A footprint whose nodes are not all still free is dropped: the stack - * simply ends, exactly as it would have before. ---*/ + /*--- Adopt what the neighbours sent, tags ascending so arrival order cannot change the outcome. + * A footprint whose nodes are not all free is dropped and the stack simply ends. ---*/ std::sort(inherited.begin(), inherited.end(), [](const CInherited& a, const CInherited& b) { return a.tag < b.tag; }); @@ -2534,20 +2451,19 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC layer0.push_back(p); } /*--- The footprint has to arrive whole and connected, the same test any other layer passes. ---*/ - if (ok && !isConnectedLayer(layer0)) ok = false; + if (ok && !IsConnectedLayer(fine_grid, layer0)) ok = false; if (ok) { std::array d0{}; for (unsigned short d = 0; d < nDim; ++d) d0[d] = inherited[i].dir[d]; /*--- An inherited layer is an interior one, so it is NOT subject to the single-layer rule the * boundary layer gets: it opens an ordinary two-deep coarse CV and waits for its partner. ---*/ - const auto nf = addFront(layer0, d0, inherited[i].tag, blockFor(layer0)); + const auto nf = addFront(layer0, d0, inherited[i].tag, BlockFor(maxAgglomSize, layer0)); for (auto p : layer0) { claimed[p] = 1; } ct[P_LAYERS]++; - ct[P_HANDIN]++; - if (pendingLayers[nf] >= nBlock[nf]) emit(nf); + if (fronts[nf].pendingLayers >= fronts[nf].nBlock) emit(nf); } i = j; } @@ -2556,49 +2472,30 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC /*--- Nothing should be left buffered, but a front retired outside the loop would strand its nodes * with a parent index that was never assigned. ---*/ - for (unsigned long f = 0; f < front.size(); ++f) emit(f); + for (unsigned long f = 0; f < fronts.size(); ++f) emit(f); - /*--- How far each front got. Fronts reaching the same height leave a flat interface with ordinary - * agglomeration; a spread here means that interface came out as a staircase. A rank with no fronts - * leaves dmin at its sentinel so it stays out of the MPI_MIN below. ---*/ + /*--- A rank with no fronts leaves dmin at its sentinel so it stays out of the MPI_MIN. ---*/ unsigned long dmin = std::numeric_limits::max(), dmax = 0; - for (unsigned long f = 0; f < front.size(); ++f) { - if (front[f].empty()) continue; - dmin = std::min(dmin, depth[f]); - dmax = std::max(dmax, depth[f]); + for (unsigned long f = 0; f < fronts.size(); ++f) { + if (fronts[f].nodes.empty()) continue; + dmin = std::min(dmin, fronts[f].depth); + dmax = std::max(dmax, fronts[f].depth); } - /*--- Summed over all ranks: rank 0's own fronts would make a partitioned run look like a fraction of - * the mesh it is not. Every rank must reach these collectives. ---*/ - unsigned long tot[P_COUNT] = {0}; + /*--- Every rank must reach these. ---*/ + unsigned long tot[P_COUNT] = {0}, pairTot[2] = {0}, depthMin = 0, depthMax = 0; + unsigned long pair[2] = {Index_CoarseCV - starting_Index_CoarseCV, seeds.node.size()}; SU2_MPI::Allreduce(ct, tot, P_COUNT, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - - unsigned long pair[2] = {Index_CoarseCV - starting_Index_CoarseCV, seeds.node.size()}, pairTot[2] = {0}; SU2_MPI::Allreduce(pair, pairTot, 2, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); - - unsigned long depthMin = 0, depthMax = 0; SU2_MPI::Allreduce(&dmin, &depthMin, 1, MPI_UNSIGNED_LONG, MPI_MIN, SU2_MPI::GetComm()); SU2_MPI::Allreduce(&dmax, &depthMax, 1, MPI_UNSIGNED_LONG, MPI_MAX, SU2_MPI::GetComm()); - if (depthMin == std::numeric_limits::max()) depthMin = 0; /*--- No fronts anywhere. ---*/ + if (depthMin == std::numeric_limits::max()) depthMin = 0; if (rank != MASTER_NODE) return {}; stringstream out; - out << " MG level " << iMesh << " paving: " << tot[P_STACKS] << " fronts from " << pairTot[1] - << " seed nodes, patch sizes "; - for (unsigned n = 1; n <= 8; ++n) - if (tot[P_HIST + n] > 0) out << n << "x" << tot[P_HIST + n] << " "; - out << "\n Coarse CVs from fronts: " << pairTot[0] << " covering " << tot[P_COVERED] << " nodes in " << tot[P_LAYERS] - << " layers, front depth " << depthMin << " to " << depthMax; - if (tot[P_SEMICV] + tot[P_FULLCV] > 0) - out << "\n Coarse CVs by depth: " << tot[P_FULLCV] << " two layers deep, " << tot[P_SEMICV] - << " one layer (top of a stack)"; - if (tot[P_HANDOUT] + tot[P_HANDIN] > 0) - out << "\n Stacks handed across partitions: " << tot[P_HANDOUT] << " sent, " << tot[P_HANDIN] << " picked up"; - if (tot[P_SPLIT] > 0) - out << "\n Footprints split at partitions: " << tot[P_SPLIT] << " cut by an interface (" << tot[P_SPLITLOC] - << " nodes marching on here, " << tot[P_SPLITHAND] << " handed across)"; - if (tot[P_SPLITDROP] > 0) out << "\n Nodes lost in split pieces that came apart: " << tot[P_SPLITDROP]; - out << "\n"; + out << " MG level " << iMesh << " paving: " << tot[P_STACKS] << " fronts from " << pairTot[1] << " seeds, " + << pairTot[0] << " CVs covering " << tot[P_COVERED] << " nodes in " << tot[P_LAYERS] << " layers, depth " + << depthMin << " to " << depthMax << "\n"; return out.str(); } diff --git a/SU2_CFD/include/integration/CMultiGridIntegration.hpp b/SU2_CFD/include/integration/CMultiGridIntegration.hpp index 8b607fb5c01..b4d552d5dcc 100644 --- a/SU2_CFD/include/integration/CMultiGridIntegration.hpp +++ b/SU2_CFD/include/integration/CMultiGridIntegration.hpp @@ -321,19 +321,15 @@ class CMultiGridIntegration final : public CIntegration { static constexpr int MAX_MG_LEVELS = 10; - /*--- Upper bound on nVar for the small per-point scratch arrays used by the restriction and - * prolongation loops, so they can live on the stack instead of being heap-allocated on every - * call. Must be >= the largest MAXNVAR of any variable class integrated by this class, - * currently CNEMOEulerVariable::MAXNVAR = 25. ---*/ static constexpr unsigned short MAXNVAR = 25; static constexpr unsigned short MAXNDIM = 3; /*--- Early-exit smoothing state (shared across OMP threads via master write + barrier). ---*/ - bool mg_early_exit_flag = false; /*!< \brief Shared flag for early exit across OMP threads. */ + bool mg_early_exit_flag = false; /*!< \brief Shared flag for early exit across OMP threads. */ passivedouble mg_initial_smooth_rms = 0.0; /*!< \brief Initial RMS residual before current smoothing phase (FAS). */ passivedouble mg_prev_smooth_rms = 0.0; /*!< \brief RMS residual from previous smoothing step; used for stagnation detection. */ - passivedouble mg_fine_rms_ema = 0.0; /*!< \brief EMA of fine-grid pre-smooth RMS across cycles; cross-cycle trend signal. */ - passivedouble last_crossCycleRatio = 1.0; /*!< \brief crossCycleRatio from the most recent cycle; stored for display only. */ + passivedouble mg_fine_rms_ema = 0.0; /*!< \brief EMA of fine-grid pre-smooth RMS across cycles; cross-cycle trend signal. */ + passivedouble last_crossCycleRatio = 1.0; /*!< \brief crossCycleRatio from the most recent cycle; stored for display only. */ /*--- Actual iteration counts per MG level, filled each cycle for the compact output summary. ---*/ unsigned short lastPreSmoothIters[MAX_MG_LEVELS+1] = {}; diff --git a/SU2_CFD/src/integration/CMultiGridIntegration.cpp b/SU2_CFD/src/integration/CMultiGridIntegration.cpp index 2f034d40ae3..5c85dabb4fb 100644 --- a/SU2_CFD/src/integration/CMultiGridIntegration.cpp +++ b/SU2_CFD/src/integration/CMultiGridIntegration.cpp @@ -1040,13 +1040,7 @@ void CMultiGridIntegration::SmoothProlongated_Correction(unsigned short RunTime_ } /*--- Refresh the halo entries of the correction with the values their owner ranks just - * computed. The next sweep averages LinSysRes over the neighbours of every domain point, - * and across a partition boundary those neighbours are halo points, so this has to run - * once per sweep rather than once at the end. It comes after the restore so that a halo - * point sitting on a physical boundary mirrors its owner's restored value. - * - * The barrier is required: the restore loop above only carries an implicit barrier for the - * markers that pass the test, so if the last marker is skipped there is none. ---*/ + * computed. ---*/ SU2_OMP_BARRIER CSysMatrixComms::Initiate(solver->LinSysRes, geometry, config); diff --git a/TestCases/euler/channel/inv_channel_RK.cfg b/TestCases/euler/channel/inv_channel_RK.cfg index 6d44b6ad5e0..f0a487a55f5 100644 --- a/TestCases/euler/channel/inv_channel_RK.cfg +++ b/TestCases/euler/channel/inv_channel_RK.cfg @@ -44,11 +44,11 @@ MARKER_MONITORING= ( upper_wall, lower_wall ) % ------------- COMMON PARAMETERS DEFINING THE NUMERICAL METHOD ---------------% % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES -CFL_NUMBER= 1.0 +CFL_NUMBER= 1 CFL_ADAPT= NO -CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) +CFL_ADAPT_PARAM= ( 0.9, 1.1, 1.0, 100.0, 0.1 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) -ITER= 110 +ITER= 1000 LINEAR_SOLVER= BCGSTAB LINEAR_SOLVER_ERROR= 1E-1 LINEAR_SOLVER_ITER= 10 @@ -57,11 +57,11 @@ LINEAR_SOLVER_ITER= 10 % MGLEVEL= 3 MGCYCLE= W_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) -MG_CORRECTION_SMOOTH= ( 4, 4, 4, 4 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) +MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % @@ -94,4 +94,4 @@ GRAD_OBJFUNC_FILENAME= of_grad SURFACE_FILENAME= surface_flow SURFACE_ADJ_FILENAME= surface_adjoint OUTPUT_WRT_FREQ= 250 -SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG) +SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG, AVG_CFL) diff --git a/TestCases/euler/wedge/inv_wedge_HLLC.cfg b/TestCases/euler/wedge/inv_wedge_HLLC.cfg index 64fb1967366..4dce47518b8 100644 --- a/TestCases/euler/wedge/inv_wedge_HLLC.cfg +++ b/TestCases/euler/wedge/inv_wedge_HLLC.cfg @@ -55,11 +55,11 @@ LINEAR_SOLVER_ITER= 3 % MGLEVEL= 3 MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/fixed_cl/naca0012/inv_NACA0012.cfg b/TestCases/fixed_cl/naca0012/inv_NACA0012.cfg index b9c7cc06667..ade966ede21 100644 --- a/TestCases/fixed_cl/naca0012/inv_NACA0012.cfg +++ b/TestCases/fixed_cl/naca0012/inv_NACA0012.cfg @@ -69,11 +69,11 @@ LINEAR_SOLVER_ITER= 3 % MGLEVEL= 3 MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/navierstokes/cylinder/cylinder_lowmach.cfg b/TestCases/navierstokes/cylinder/cylinder_lowmach.cfg index 1c175581e3a..402b8c9158b 100644 --- a/TestCases/navierstokes/cylinder/cylinder_lowmach.cfg +++ b/TestCases/navierstokes/cylinder/cylinder_lowmach.cfg @@ -42,7 +42,7 @@ MARKER_MONITORING= ( cylinder ) % ------------- COMMON PARAMETERS DEFINING THE NUMERICAL METHOD ---------------% % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES -CFL_NUMBER= 4.0 +CFL_NUMBER= 100.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) @@ -58,12 +58,12 @@ LINEAR_SOLVER_ITER= 3 % -------------------------- MULTIGRID PARAMETERS -----------------------------% % MGLEVEL= 3 -MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MGCYCLE= W_CYCLE +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/navierstokes/cylinder/lam_cylinder.cfg b/TestCases/navierstokes/cylinder/lam_cylinder.cfg index 70f8692f64b..8d696643fbf 100644 --- a/TestCases/navierstokes/cylinder/lam_cylinder.cfg +++ b/TestCases/navierstokes/cylinder/lam_cylinder.cfg @@ -42,7 +42,7 @@ MARKER_MONITORING= ( cylinder ) % ------------- COMMON PARAMETERS DEFINING THE NUMERICAL METHOD ---------------% % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES -CFL_NUMBER= 500.0 +CFL_NUMBER= 1000.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) @@ -59,11 +59,11 @@ LINEAR_SOLVER_ITER= 3 % MGLEVEL= 3 MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % diff --git a/TestCases/navierstokes/flatplate/lam_flatplate.cfg b/TestCases/navierstokes/flatplate/lam_flatplate.cfg index f213b10a0a7..0be7e3573cf 100644 --- a/TestCases/navierstokes/flatplate/lam_flatplate.cfg +++ b/TestCases/navierstokes/flatplate/lam_flatplate.cfg @@ -60,11 +60,11 @@ LINEAR_SOLVER_ITER= 3 % MGLEVEL= 3 MGCYCLE= V_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_PRE_SMOOTH= ( 2, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % @@ -92,3 +92,4 @@ RESTART_ADJ_FILENAME= restart_adj GRAD_OBJFUNC_FILENAME= of_grad OUTPUT_WRT_FREQ= 250 SCREEN_OUTPUT= (INNER_ITER, RMS_DENSITY, RMS_ENERGY, DRAG, SURFACE_MASSFLOW, TOTAL_HEATFLUX, TOTAL_HEATFLUX_ON_SURFACE) +WRT_PERFORMANCE= YES From f2cc8fb594630ca10caf75233dd88e25d1e372de Mon Sep 17 00:00:00 2001 From: bigfooted Date: Tue, 8 Sep 2026 23:38:52 +0200 Subject: [PATCH 54/54] cleanup large comments --- Common/src/geometry/CMultiGridGeometry.cpp | 393 +++++++-------------- TestCases/rans/rae2822/turb_SA_RAE2822.cfg | 20 +- 2 files changed, 140 insertions(+), 273 deletions(-) diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp index bd1877de9a4..3a52fa1aee4 100644 --- a/Common/src/geometry/CMultiGridGeometry.cpp +++ b/Common/src/geometry/CMultiGridGeometry.cpp @@ -46,9 +46,6 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un 3rd) One marker ---> Surface (always agglomerate) 4th) No marker ---> Internal Volume (always agglomerate) ---*/ - // Note that for MPI, we introduce interfaces and we can choose to have agglomeration over - // the interface or not. Nishikawa chooses not to agglomerate over interfaces. - /*--- Set a marker to indicate indirect agglomeration, for quads and hexs, i.e. consider up to neighbors of neighbors. For other levels this information is propagated down during their construction. ---*/ @@ -88,10 +85,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- STEP 0: pave the domain with advancing fronts rising from the boundaries, wall CV included. - * It runs before the boundary agglomeration so that the wall CV and the layers above it share one - * footprint. Everything it claims is marked agglomerated, so the passes below skip it. The coarse - * CVs it creates occupy [firstLineCV, endLineCV), which is how the repair passes identify them. ---*/ + /*--- STEP 0: pave the domain with advancing fronts rising from the boundaries. The coarse CVs it + * creates occupy [firstLineCV, endLineCV). ---*/ const auto firstLineCV = Index_CoarseCV; if (config->GetMGOptions().MG_Implicit_Lines) { pavingReport = AgglomerateImplicitLines(Index_CoarseCV, fine_grid, config, iMesh); @@ -113,8 +108,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- Skip periodic boundaries: do not agglomerate on periodic markers. ---*/ if (config->GetMarker_All_KindBC(iMarker) == PERIODIC_BOUNDARY) continue; - /*--- Skip SEND_RECEIVE markers: they record that a point is mirrored on another rank, not that it - * lies on a boundary. Such points are left to the domain pass, as in a serial run. ---*/ + /*--- Skip SEND_RECEIVE markers, those points are left to the domain pass. ---*/ if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { @@ -148,8 +142,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker(); jMarker++) { if (config->GetMarker_All_KindBC(jMarker) == SEND_RECEIVE) continue; if (fine_grid->nodes->GetVertex(iPoint, jMarker) != -1) { - /*--- Count every physical marker, the counter > 2 test needs the true count, but only - store the first few, which is all the matching rules ever look at. ---*/ + /*--- Count every physical marker, but store only the first few. ---*/ if (counter < 3) copy_marker[counter] = jMarker; counter++; @@ -193,9 +186,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- Two physical markers meet here. ---*/ if (counter == 2) { - /*--- In 2D that is a genuine corner in the geometry, which is never agglomerated. A wall - point merely split by a partition interface no longer reaches this branch: it counts - one physical marker and is handled above as the valley point it is. ---*/ + /*--- In 2D that is a corner in the geometry, which is never agglomerated. ---*/ if (nDim == 2) agglomerate_seed = false; /*--- In 3D, this is a ridge point (an edge where two surface markers meet). ---*/ if (nDim == 3) agglomerate_seed = true; @@ -227,9 +218,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (counter > 2) agglomerate_seed = false; - /*--- ...and so is a node where two markers of DIFFERENT type meet, whatever the count and - * whatever the dimension. A coarse CV holding such a node would average two conditions the - * fine grid applies separately. ---*/ + /*--- ...and so is a node where two markers of different type meet, in any dimension. ---*/ if (mixedBC[iPoint]) agglomerate_seed = false; /*--- If the seed (parent) can be agglomerated, we try to agglomerate connected childs to the parent ---*/ @@ -305,9 +294,7 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un i.e. make one coarse CV with a single child. ---*/ for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { - /*--- As in STEP 1, a SEND_RECEIVE marker does not make a point a boundary point. Turning the - leftovers of those markers into single-child coarse CVs here would strand every interior point - along a partition interface before the domain pass below ever gets to see it. ---*/ + /*--- As in STEP 1, a SEND_RECEIVE marker does not make a point a boundary point. ---*/ if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; for (auto iVertex = 0ul; iVertex < fine_grid->GetnVertex(iMarker); iVertex++) { @@ -347,9 +334,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un vector members, candidates; members.reserve(maxAgglomSize); - /*--- A local frame at the seed: up to nDim of its incident edges, as mutually orthogonal as possible, - * each with its own length. An offset is resolved onto these and each component divided by that - * direction's spacing. ---*/ + /*--- A local frame at the seed: up to nDim of its incident edges, as mutually orthogonal as + * possible, each with its own length. ---*/ vector> frameDir, edgeDir; vector frameLen, edgeLen; vector edgeUsed; @@ -410,9 +396,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un /*--- A seed with no usable edge cannot be measured against anything, leave the scale at one. ---*/ if (seedShortest == std::numeric_limits::max()) seedShortest = 1.0; - /*--- The shortest edge goes in first, so that the direction the mesh is stretched along is the - * one measured against its own small spacing; the rest are taken in order of how orthogonal - * they are to what is already in the frame. ---*/ + /*--- The shortest edge goes in first, the rest in order of how orthogonal they are to what is + * already in the frame. ---*/ frameDir.clear(); frameLen.clear(); edgeUsed.assign(edgeDir.size(), 0); @@ -463,8 +448,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un unsigned short shared = 0; for (auto jPoint : fine_grid->nodes->GetPoints(CVPoint)) shared += inCV[jPoint]; - /*--- Distance to the centroid in cells rather than in metres: the offset is scaled by the - * seed edge pointing most nearly along it. ---*/ + /*--- Distance to the centroid in cells: the offset is scaled by the seed edge pointing + * most nearly along it. ---*/ su2double off[MAXNDIM] = {0.0}; for (unsigned short d = 0; d < nDim; ++d) off[d] = fine_grid->nodes->GetCoord(CVPoint)[d] - centroid[d]; const su2double offLen = GeometryToolbox::Norm(nDim, off); @@ -528,21 +513,13 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un SetPoint_Connectivity(fine_grid); - /*--- The connectivity just built only knows about coarse CVs of this rank: the halo CVs do not - exist until the MPI relay below runs, and the relay cannot run earlier because it broadcasts the - parent indices that the merge here is still free to change. So a CV touching a partition boundary - may look isolated while actually having neighbors on the other rank, and merging it would be - wrong. Mark those CVs and leave them alone; a genuinely isolated CV in the interior is unaffected. - Note this deliberately keeps the conservative outcome the sentinel used to produce by accident, - but only for the CVs that really do border another rank rather than for every CV near one. ---*/ + /*--- The connectivity just built only knows about coarse CVs of this rank, so a CV touching a + partition boundary may look isolated. Mark those CVs and leave them alone. ---*/ /*--- Coarse CVs to leave exactly as the agglomeration made them: those holding a node where two - * different boundary conditions meet. Both repair passes remove one-child CVs, and a deliberately - * isolated junction is one, so they must be protected as a merge TARGET as well as a source. ---*/ + * different boundary conditions meet. ---*/ vector mustStayAlone(nPointDomain, false); - /*--- ...and which coarse CVs hold a boundary node at all. A boundary node is never agglomerated with - * an interior one, which the paving already respects. The repair passes below would undo it from - * the other end, since a boundary CV's neighbours include the interior CV sitting on top of it. ---*/ + /*--- ...and which coarse CVs hold a boundary node at all. ---*/ vector cvOnBoundary(nPointDomain, false); for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) for (auto iChildren = 0u; iChildren < nodes->GetnChildren_CV(iCoarsePoint); iChildren++) { @@ -551,10 +528,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (onPhysBoundary[iFinePoint]) cvOnBoundary[iCoarsePoint] = true; } - /*--- Which physical boundaries each coarse CV sits on, one bit per marker. cvOnBoundary records only - * THAT a CV touches a boundary, so a one-child CV on boundary A could be merged into a neighbour on - * boundary B; SetVertex would then give the merged CV marker A while its body sits on B. Comparing - * marker sets is strictly stronger and keeps a merge inside one boundary. ---*/ + /*--- Which physical boundaries each coarse CV sits on, one bit per marker. Comparing marker sets + * keeps a merge inside one boundary. ---*/ vector cvMarkerMask(nPointDomain, 0); { vector bitOfMarker(fine_grid->GetnMarker(), -1); @@ -581,10 +556,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- A boundary CV built by the paving is the base of a stack and must keep its footprint. Merging - * one into a neighbour leaves the column headless and widens the base, which on the next level - * stops it seeding a front and lets an interior front swallow it mid-stack. The repair passes - * below are for the interior singleton left where a line narrows, not for these. ---*/ + /*--- A boundary CV built by the paving is the base of a stack and keeps its footprint, so the + * repair passes below leave it alone. ---*/ auto isStackBase = [&](unsigned long iCoarsePoint) { return cvOnBoundary[iCoarsePoint] && (iCoarsePoint >= firstLineCV) && (iCoarsePoint < endLineCV); }; @@ -678,31 +651,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un } } - /*--- The pass above only rescues a coarse CV that has a single coarse neighbor, i.e. one enclosed - entirely within another. It does nothing for a CV that was left with a single fine-grid child but - sits between several coarse neighbors, which happens routinely with the implicit-line stacks above: - a bundle whose members reach the top of the boundary layer at slightly different heights (graded - spacing, local curvature) narrows one line at a time (see AgglomerateImplicitLines, Phase C), and - the fine node just past where a line dropped out is left to fend for itself. It usually has no - unclaimed neighbor left to agglomerate with by the time the ordinary domain pass (STEP 2) reaches - it, so it becomes a coarse CV of its own, surrounded on multiple sides by CVs it cannot join under - the single-neighbor rule above. Left alone, this is exactly the "hole" that turns what should be a - compact block into an L shape with an isolated singleton sitting in the missing corner. Merge such - a CV into whichever neighbor currently has the fewest children (and still has room under - maxAgglomSize): the smallest neighbor is the one most likely to be the under-filled block the - singleton belongs with, e.g. the 3-node L that this merge completes into a 4-node square, and - folding into it keeps the coarse grid from accumulating disproportionately large CVs. - - Unlike the pass above this one deliberately does not skip CVs that border another rank. That guard - exists because "has exactly one coarse neighbour" is measured on connectivity this rank cannot yet - see in full, so it misfires on a CV whose other neighbours merely live across the partition. The - trigger here is a child count, which is complete locally: a CV owning one fine point owns one fine - point no matter how the mesh was cut. The merge is local too - every candidate neighbour comes from - the connectivity built above and is therefore an owned CV, every child is an owned point, and the - MPI relay below broadcasts the result afterwards, so the ranks stay consistent. Requiring at least - two local neighbours below also rules out the one case that could not be repaired locally, a CV - whose neighbours are all halo: a halo CV mirrors another rank's decision and must not be added to - here. Keeping the guard cost 455 of 473 unrepaired singletons on a four-rank 3D bump. ---*/ + /*--- Merge a coarse CV that still holds a single fine child into whichever coarse neighbor has the + fewest children. Both the merged CV and the target are owned by this rank. ---*/ for (auto iCoarsePoint = 0ul; iCoarsePoint < nPointDomain; iCoarsePoint++) { if (nodes->GetnChildren_CV(iCoarsePoint) != 1) continue; @@ -710,12 +660,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un if (isStackBase(iCoarsePoint)) continue; if (nodes->GetnPoint(iCoarsePoint) <= 1) continue; /*--- Already handled above, or truly islanded. ---*/ - /*--- The smallest neighbour is the one most likely to be an under-filled block, and taking the - smallest also means a neighbour with room is always preferred over a full one: anything below - maxAgglomSize necessarily has fewer children than anything at it. When every neighbour is full - the smallest is taken anyway, one child over the limit. maxAgglomSize is a quality target rather - than a capacity - Children_CV grows on demand - and a CV of nine is a far smaller blemish on the - coarse grid than the CV of one it removes. ---*/ + /*--- Pick the neighbour with the fewest children. When every neighbour is already at + maxAgglomSize the smallest is still taken, one child over the limit. ---*/ unsigned long best_neighbor = std::numeric_limits::max(); unsigned short best_nChildren = 0; for (auto jCoarsePoint : nodes->GetPoints(iCoarsePoint)) { @@ -739,19 +685,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->SetnChildren_CV(iCoarsePoint, 0); } - /*--- Compact the coarse numbering. Both repair passes empty a control volume by moving its children - elsewhere, but nPointDomain was fixed before them, so an emptied index survives as a control volume - that owns no fine points at all. It ends up with EPS volume, (0,0,0) coordinates and, because - neighbours are derived through children, no neighbours either. Nothing is then adjacent to it on the - next level, so it cannot be agglomerated there and becomes a one-child CV again, and so on down the - hierarchy: the repair would otherwise trade a bad CV on this level for a degenerate one on every - level above. Squeezing the empty slots out keeps the CV count honest and stops that propagation. - - Renumbering is safe here because only three things refer to coarse indices at this point - the - children lists, the indirect-agglomeration flags, and the fine grid's parent indices - and the MPI - relay below has not run yet, so no other rank has seen these numbers. Only parents of *owned* fine - points are remapped; halo points are assigned by the relay afterwards. Every surviving parent keeps - a valid new index because a CV that still has a child is never removed. ---*/ + /*--- Compact the coarse numbering, squeezing out the indices the repair passes emptied. The + children lists, indirect-agglomeration flags and owned parent indices are remapped. ---*/ { constexpr auto NO_INDEX = std::numeric_limits::max(); @@ -791,11 +726,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un nodes->ResetPoints(); #ifdef HAVE_MPI - /*--- Reset halo point parents before MPI agglomeration. - When creating level N from level N-1, the fine grid (level N-1) - already has Parent_CV set from when it was created from level N-2. - Those parent indices point to level N, but when creating level N+1, they would be - incorrectly interpreted as level N+1 indices. ---*/ + /*--- Reset halo point parents before MPI agglomeration, the fine grid still carries the parent + indices it was given when it was itself built. ---*/ for (auto iPoint = fine_grid->GetnPointDomain(); iPoint < fine_grid->GetnPoint(); iPoint++) { fine_grid->nodes->SetParent_CV(iPoint, std::numeric_limits::max()); @@ -856,8 +788,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un vector Parent_Local(nVertexR); vector Children_Local(nVertexR); - /*--- First pass: Determine which parents will actually be used (have non-skipped children). - This prevents creating orphaned halo CVs that have coordinates (0,0,0). ---*/ + /*--- First pass: determine which parents will actually be used, i.e. have non-skipped + children. ---*/ vector parent_used(Aux_Parent.size(), false); vector parent_local_index(Aux_Parent.size(), std::numeric_limits::max()); @@ -952,10 +884,8 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un const su2double ratio = su2double(Global_nPointFine) / su2double(Global_nPointCoarse); - /*--- Stop coarsening once the smallest per-rank partition falls below the minimum, - not just the summed total, since each rank runs its own MG hierarchy locally - and a partition that agglomerates down to too few CVs degenerates the operator - on that rank even if other ranks still have plenty of points. ---*/ + /*--- Stop coarsening once the smallest per-rank partition falls below the minimum, not just the + summed total. ---*/ if (Min_nPointCoarse < config->GetMGOptions().MG_Min_MeshSize) { if (rank == MASTER_NODE) cout << "MG level " << iMesh << " has only " << Min_nPointCoarse @@ -1043,9 +973,7 @@ vector CMultiGridGeometry::FindMixedBoundaryNodes(const CGeometry* fine_gr vector mixed(fine_grid->GetnPoint(), 0); /*--- The first physical condition seen at each node, -1 until one is. A second one of a different - * KIND is what makes the node mixed. A second marker of the same kind - two wall patches meeting - * - is not, and neither is SEND_RECEIVE, which records where the partition runs and says nothing - * about the boundary condition. ---*/ + * kind is what makes the node mixed. ---*/ vector firstBC(fine_grid->GetnPoint(), -1); for (auto iMarker = 0u; iMarker < fine_grid->GetnMarker(); iMarker++) { @@ -1081,11 +1009,8 @@ bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vectornodes->GetBoundary(CVPoint)) { - /*--- Identify the physical markers of the vertex that we want to agglomerate. SEND_RECEIVE - markers are skipped for the same reason as on the seed side: they say nothing about the - boundary condition the candidate carries, only that it is mirrored on another rank. A - candidate whose markers are all SEND_RECEIVE therefore ends up with counter == 0 and is - rejected below, which is the answer a serial run gives for the interior point it really is. ---*/ + /*--- Identify the physical markers of the vertex that we want to agglomerate. A candidate whose + markers are all SEND_RECEIVE ends up with counter == 0 and is rejected below. ---*/ for (auto jMarker = 0u; jMarker < fine_grid->GetnMarker(); jMarker++) { if (config->GetMarker_All_KindBC(jMarker) == SEND_RECEIVE) continue; @@ -1105,10 +1030,6 @@ bool CMultiGridGeometry::SetBoundAgglomeration(unsigned long CVPoint, vector& Suitable_Indirect_Neighbors, unsigned long iPoint, unsigned long Index_CoarseCV, const CGeometry* fine_grid) const { /*--- Create a list with the first neighbors, including the seed. ---*/ @@ -1200,11 +1119,8 @@ void CMultiGridGeometry::SetPoint_Connectivity(const CGeometry* fine_grid) { /*--- loop over the parent CVs (coarse grid) of its (fine) neighbors. ---*/ for (auto iFinePoint_Neighbor : fine_grid->nodes->GetPoints(iFinePoint)) { const auto iParent = fine_grid->nodes->GetParent_CV(iFinePoint_Neighbor); - /*--- Skip neighbors whose parent is not known yet. The first call to this function happens - during construction, before the MPI relay has assigned parents to the fine grid's halo - points, so those still hold the sentinel. Letting it through would add one fake neighbor to - every coarse CV along a partition boundary, which both corrupts nNeighbor and hides the CV - from the isolated-CV repair. The driver calls this again once the relay has run. ---*/ + /*--- Skip neighbors whose parent is not known yet, halo points still hold the sentinel until + the MPI relay has run. ---*/ if (iParent == std::numeric_limits::max()) continue; /*--- If it is not the target coarse point, it is a coarse neighbor. ---*/ if (iParent != iCoarsePoint) { @@ -1537,8 +1453,7 @@ void CMultiGridGeometry::FindNormal_Neighbor(const CConfig* config) { su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, unsigned long iPoint, unsigned short iMarker) const { - /*--- Compute local curvature (maximum angle between adjacent face normals) at a boundary vertex. - This is used to determine if agglomeration is safe based on a curvature threshold. ---*/ + /*--- Local curvature is the maximum angle between adjacent face normals at a boundary vertex. ---*/ /*--- Get the vertex index for this point on this marker ---*/ long iVertex = fine_grid->nodes->GetVertex(iPoint, iMarker); @@ -1600,10 +1515,8 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid, } CMultiGridGeometry::CNodeStiffness CMultiGridGeometry::ComputeNodeStiffness(const CGeometry* fine_grid) const { - /*--- Coupling across the dual face between a node and a neighbour. For a cell of size dx by dy this - * is 1/dy across the wall-normal face and 1/dx across the tangential one, so the ratio of largest - * to smallest weight at a node is the local aspect ratio. Unlike CGeometry::Aspect_Ratio this is - * available on every multigrid level, since SetControlVolume builds the dual grid on each. ---*/ + /*--- Coupling across the dual face between a node and a neighbour, so the ratio of largest to + * smallest weight at a node is the local aspect ratio. ---*/ const auto nPointFine = fine_grid->GetnPoint(); CNodeStiffness stiff; @@ -1641,7 +1554,7 @@ CMultiGridGeometry::CNodeStiffness CMultiGridGeometry::ComputeNodeStiffness(cons namespace { /*--- Unit normal of a boundary at a vertex, false if the marker does not reach iPoint. Boundary - * normals point INTO the domain, so this doubles as a starting direction to march in. ---*/ + * normals point into the domain. ---*/ bool VertexUnitNormal(const CGeometry* grid, unsigned short nDim, unsigned long iPoint, unsigned short iMarker, su2double* unitNormal) { const long iVertex = grid->nodes->GetVertex(iPoint, iMarker); @@ -1659,8 +1572,7 @@ bool IsAdjacent(const CGeometry* grid, unsigned long a, unsigned long b) { return std::find(pts.begin(), pts.end(), b) != pts.end(); } -/*--- Is a footprint one connected patch? A set that falls into pieces is not the extrusion of - * anything, so an arriving footprint and a piece left by a split both have to pass this. ---*/ +/*--- Is a footprint one connected patch? ---*/ bool IsConnectedLayer(const CGeometry* grid, const vector& layer) { if (layer.size() < 2) return true; vector seen(layer.size(), 0); @@ -1680,8 +1592,8 @@ bool IsConnectedLayer(const CGeometry* grid, const vector& layer) return nSeen == layer.size(); } -/*--- Is the new layer topologically identical to the old? They are index-aligned, so phi maps - * old[k] to new[k], and the layer is valid when phi is an isomorphism of the induced subgraphs. ---*/ +/*--- Is the new layer topologically identical to the old? The layers are index-aligned, so the map + * old[k] to new[k] has to be an isomorphism of the induced subgraphs. ---*/ bool LayerIsIsomorphic(const CGeometry* grid, const vector& oldL, const vector& newL) { const auto n = oldL.size(); if (newL.size() != n) return false; @@ -1704,8 +1616,7 @@ bool LayerIsIsomorphic(const CGeometry* grid, const vector& oldL, return true; } -/*--- A boundary node ends a front only if the step runs INTO it, i.e. roughly along its normal. One - * running ALONG a boundary is a legitimate interior node of the stack. ---*/ +/*--- Does the step run into a boundary at jPoint, i.e. roughly along that boundary's normal? ---*/ bool EntersBoundary(const CGeometry* grid, const CConfig* config, unsigned short nDim, unsigned long jPoint, const su2double* stepDir, su2double cosBoundary) { for (unsigned short iMarker = 0; iMarker < grid->GetnMarker(); iMarker++) { @@ -1717,15 +1628,14 @@ bool EntersBoundary(const CGeometry* grid, const CConfig* config, unsigned short return false; } -/*--- Fine layers the next coarse CV of this front holds: two, so the stack coarsens by the same - * factor along the marching direction as the footprint does across it. One only if a second - * layer would exceed the agglomeration size limit. ---*/ +/*--- Fine layers the next coarse CV of this front holds: two, or one if that would exceed the + * agglomeration size limit. ---*/ unsigned long BlockFor(short int maxAgglomSize, const vector& layer) { return (layer.size() * 2 > static_cast(maxAgglomSize)) ? 1 : 2; } -/*--- Rank-independent name for a set of nodes: the smallest global index in it, +1 so that 0 is - * free to mean "nothing". A footprint is claimed by one front and no other, so it is unique. ---*/ +/*--- Rank-independent name for a set of nodes: the smallest global index in it, +1 so that 0 means + * "nothing". ---*/ unsigned long TagOfSet(const CGeometry* grid, const vector& set) { unsigned long t = std::numeric_limits::max(); for (auto p : set) t = std::min(t, grid->nodes->GetGlobalIndex(p)); @@ -1739,10 +1649,8 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet /*--- Fraction of a marker's nodes that must sit in a layer before the whole marker may seed. ---*/ constexpr passivedouble QUALIFIED_FRACTION = 0.5; constexpr passivedouble ANGLE_THRESHOLD_DEG = 30.0; - /*--- Smallest local cell aspect ratio for which a node still counts as part of a stretched layer. - * Not exposed as an option: the two populations it separates are decades apart - a boundary - * carrying a layer normal to itself measures in the hundreds or more, one in isotropic mesh - * measures about one - so every value between them selects the same set of markers. ---*/ + /*--- Smallest local cell aspect ratio for which a node still counts as part of a stretched + * layer. ---*/ constexpr passivedouble MIN_AR = 2.0; const su2double cos_threshold = cos(ANGLE_THRESHOLD_DEG * PI_NUMBER / 180.0); @@ -1757,8 +1665,7 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet }; /*--- True if the mesh at iPoint is stretched along the boundary normal, i.e. this boundary has a - * layer growing off it the way a viscous wall does. On the side planes of a bump the mesh is just - * as stretched, but in a direction running ALONG the plane, and those nodes belong to the wall. ---*/ + * layer growing off it the way a viscous wall does. ---*/ auto hasLayerNormalTo = [&](unsigned long iPoint, const su2double* unitNormal) { const auto jStiffest = stiff.jStiffest[iPoint]; if (jStiffest == NO_POINT) return false; @@ -1791,24 +1698,19 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet } }; - /*--- Viscous walls always carry a stretched layer, so they seed unconditionally. Running them first - * also settles nodes where a wall meets another boundary: the wall claims them. ---*/ + /*--- Viscous walls always carry a stretched layer, so they seed unconditionally and first, which + * gives them the nodes where a wall meets another boundary. ---*/ for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) if (isWall(config->GetMarker_All_KindBC(iMarker))) seedMarker(iMarker, false); - /*--- Non-wall boundaries that still carry a layer normal to themselves, such as a symmetry plane in - * the same surface as a wall. The verdict is per marker, not per node: seeding isolated qualifying - * nodes scatters one-node patches that do not coarsen tangentially at all. ---*/ - /*--- Counted per configuration-file marker, not per local marker: ranks agree on neither the - * number nor the order of local markers, because each partition appends its own SEND_RECEIVE - * markers, so the same index means a different boundary elsewhere. ---*/ + /*--- Non-wall boundaries that still carry a layer normal to themselves, counted per + * configuration-file marker because local marker indices differ between ranks. ---*/ const auto nMarkerCfg = config->GetnMarker_CfgFile(); vector nValid(nMarkerCfg, 0), nQualified(nMarkerCfg, 0); auto canSeed = [&](unsigned short iMarker) { const auto bc = config->GetMarker_All_KindBC(iMarker); - /*--- Periodic boundaries are left out: the two halves are the same physical location under a - * transform and have their own matching, which a front running into one would disturb. ---*/ + /*--- Periodic boundaries are left out, they have their own matching. ---*/ return (bc != SEND_RECEIVE) && (bc != PERIODIC_BOUNDARY) && !isWall(bc); }; @@ -1829,8 +1731,8 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet } } - /*--- A marker is generally split over several ranks, so the verdict must be taken on all of it. - * Every rank reaches these collectives, including one that owns no boundary at all. ---*/ + /*--- A marker is generally split over several ranks, so the verdict is taken on all of it. Every + * rank reaches these collectives. ---*/ if (nMarkerCfg > 0) { vector tmp(nMarkerCfg); SU2_MPI::Allreduce(nValid.data(), tmp.data(), nMarkerCfg, MPI_UNSIGNED_LONG, MPI_SUM, SU2_MPI::GetComm()); @@ -1853,15 +1755,13 @@ CMultiGridGeometry::CFrontSeeds CMultiGridGeometry::SeedFrontNodes(const CGeomet vector> CMultiGridGeometry::BuildFrontPatches(const CFrontSeeds& seeds, const CGeometry* fine_grid, const CConfig* config, const vector& mixedBC) const { - /*--- Repeated pairwise matching, one round per doubling, partitions the seeds into compact patches: - * a boundary edge in 2D, a boundary quadrilateral in 3D. Matching keeps membership mutually - * exclusive, which choosing neighbours per seed would not. It needs only point connectivity, so it - * works on every multigrid level. This patch alone fixes the footprint of the stack above it. ---*/ + /*--- Repeated pairwise matching, one round per doubling, partitions the seeds into compact + * patches: a boundary edge in 2D, a boundary quadrilateral in 3D. ---*/ const auto nSeeds = seeds.node.size(); const unsigned long max_group = (nDim == 2) ? 2 : 4; - /*--- Physical markers each seed lies on, ascending. Seeds may only be matched when these agree, so - * a patch never straddles a change of boundary condition. ---*/ + /*--- Physical markers each seed lies on, ascending. Seeds may only be matched when these + * agree. ---*/ const auto nMarkerFine = fine_grid->GetnMarker(); vector> sig(nSeeds); for (unsigned long si = 0; si < nSeeds; ++si) @@ -1881,9 +1781,7 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront if ((sj >= 0) && (static_cast(sj) != si)) adj[si].push_back(static_cast(sj)); } - /*--- Global point index of each seed, used everywhere below as the deterministic sort key. Local - * indices depend on the partitioning, so ordering by them would make the coarse grid depend on - * the rank count. ---*/ + /*--- Global point index of each seed, used below as the partitioning-independent sort key. ---*/ vector sgkey(nSeeds); for (unsigned long si = 0; si < nSeeds; ++si) sgkey[si] = fine_grid->nodes->GetGlobalIndex(seeds.node[si]); @@ -1899,11 +1797,8 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront const unsigned nRounds = (max_group <= 1) ? 0 : ((max_group <= 2) ? 1 : 2); - /*--- One admissible merge of two groups, weighted by how many seed-to-seed adjacencies they share. - * A group lying ALONGSIDE this one touches it along its whole length and shares two, whereas one - * continuing in the same direction touches at an end and shares one. So weight 2 is the square - * and weight 1 is the strip, and the strip extrudes into a coarse CV elongated in one - * boundary-tangential direction. ---*/ + /*--- One admissible merge of two groups, weighted by how many seed-to-seed adjacencies they + * share: 2 for a group lying alongside, 1 for one continuing in the same direction. ---*/ struct CMerge { unsigned long g, h; /*!< \brief The two groups, g < h. */ unsigned long weight; /*!< \brief Shared adjacencies: 2 makes a square, 1 makes a strip. */ @@ -1923,13 +1818,12 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront for (unsigned long g = 0; g < nGroups; ++g) for (auto si : groups[g]) gkey[g] = std::min(gkey[g], sgkey[si]); - /*--- Every merge this round could make. Counted once per unordered pair: adjacency is symmetric, - * so the count from g's side equals the count from h's, and taking only h > g avoids both. ---*/ + /*--- Every merge this round could make, counted once per unordered pair by taking only + * h > g. ---*/ merges.clear(); for (unsigned long g = 0; g < nGroups; ++g) { - /*--- A node where two different boundary conditions meet stays a patch of its own, so the front - * rising from it is one node wide. Both sides of a merge are tested: skipping only the mixed - * group would still let an ordinary group reach out and take it. ---*/ + /*--- A node where two different boundary conditions meet stays a patch of its own, so both + * sides of a merge are tested for it. ---*/ if (mixedBC[seeds.node[groups[g].front()]]) continue; touched.clear(); for (auto si : groups[g]) @@ -1947,9 +1841,8 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront } } - /*--- Best merges first over all groups at once. Sweeping groups in index order lets one with no - * square partner take a weight-1 merge and consume a group a later one needed, and the failures - * cascade. Ordering globally makes every square before the first strip is considered. ---*/ + /*--- Best merges first over all groups at once, so every square is considered before the first + * strip. ---*/ std::sort(merges.begin(), merges.end(), [](const CMerge& a, const CMerge& b) { if (a.weight != b.weight) return a.weight > b.weight; if (a.keyG != b.keyG) return a.keyG < b.keyG; @@ -1981,35 +1874,30 @@ vector> CMultiGridGeometry::BuildFrontPatches(const CFront string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseCV, const CGeometry* fine_grid, const CConfig* config, unsigned short iMesh) { - /*--- Paving by advancing fronts. Each boundary patch rises into the domain keeping its footprint, so - * a coarse CV never spans two patches. A front stops at a boundary or when the next layer is not - * isomorphic to the current one. Each coarse CV is the footprint two layers deep. ---*/ + /*--- Paving by advancing fronts. Each boundary patch rises into the domain keeping its footprint, + * stopping at a boundary or where the next layer is not isomorphic to the current one. ---*/ const auto starting_Index_CoarseCV = Index_CoarseCV; const auto nPointFine = fine_grid->GetnPoint(); const auto nMarkerFine = fine_grid->GetnMarker(); constexpr auto NO_POINT = std::numeric_limits::max(); const short int maxAgglomSize = (nDim == 2) ? 4 : 8; - /*--- How nearly parallel a step must be to a boundary's normal to count as running INTO that - * boundary rather than along it. This is not a limit on where a front may go - it is only how - * "the front has reached a boundary" is recognised. ---*/ + /*--- How nearly parallel a step must be to a boundary's normal to count as running into that + * boundary rather than along it. ---*/ constexpr passivedouble BOUNDARY_ALIGN_DEG = 30.0; const su2double cos_boundary = cos(BOUNDARY_ALIGN_DEG * PI_NUMBER / 180.0); - /*--- Weight of the new step direction when the front's marching direction is updated. The direction - * only ever RANKS candidates, it never rejects one, so this is a preference and not a limit. ---*/ + /*--- Weight of the new step direction when the front's marching direction is updated. ---*/ constexpr passivedouble DIR_BLEND = 0.5; const auto stiff = ComputeNodeStiffness(fine_grid); - /*--- PHASE 1. SeedFrontNodes must be reached by every rank, including one that owns no boundary: - * it takes a collective to agree on which markers carry a layer. ---*/ + /*--- PHASE 1. SeedFrontNodes is collective and must be reached by every rank. ---*/ const auto seeds = SeedFrontNodes(fine_grid, config, stiff); const auto mixedBC = FindMixedBoundaryNodes(fine_grid, config); const auto patches = BuildFrontPatches(seeds, fine_grid, config, mixedBC); - /*--- Nodes on a boundary carrying a boundary condition; a front must not grow into one or the stack - * would straddle two boundaries. CPoint's Boundary flag is also set by SEND_RECEIVE, so it would - * stop every front one layer short of a partition. ---*/ + /*--- Nodes on a boundary carrying a boundary condition, which a front must not grow into. CPoint's + * Boundary flag cannot be used, it is also set by SEND_RECEIVE. ---*/ vector onPhysicalBoundary(nPointFine, 0); for (auto iMarker = 0u; iMarker < nMarkerFine; iMarker++) { if (config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) continue; @@ -2031,8 +1919,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC su2double dir[MAXNDIM]; /*!< Unit step direction, reused to update the front's direction. */ }; - /*--- One advancing front. Fronts are not a fixed set: one handed over from a neighbouring rank is - * appended while the rounds are running, so the loops below are bounded by fronts.size(). ---*/ + /*--- One advancing front. Fronts handed over from a neighbouring rank are appended while the + * rounds are running, so the loops below are bounded by fronts.size(). ---*/ struct CFront { vector nodes; /*!< \brief Current footprint. */ vector pending; /*!< \brief Nodes buffered for the coarse CV being built. */ @@ -2040,8 +1928,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC vector prop; /*!< \brief This round's proposed successors. */ std::array dir{}; /*!< \brief Marching direction. */ unsigned long tag = 0; /*!< \brief Rank-independent name, see TagOfSet. */ - unsigned long handTag = 0; /*!< \brief Name the handed-over piece travels under. After a - * split this is NOT tag: the pieces are separate stacks. */ + unsigned long handTag = 0; /*!< \brief Name the handed-over piece travels under, which + * after a split differs from tag. */ unsigned long depth = 0; /*!< \brief Layers laid. */ unsigned long nBlock = 0; /*!< \brief Fine layers the next coarse CV holds. */ unsigned long pendingLayers = 0; /*!< \brief Layers currently buffered. */ @@ -2065,9 +1953,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC vector claimed(nPointFine, 0); - /*--- Best free neighbour of n to step onto, ranked by alignment with dir. Local and halo candidates - * are ranked separately: a halo node cannot be claimed here, but it is where the stack would go - * next, so it is what gets handed over. ---*/ + /*--- Best free neighbour of n to step onto, ranked by alignment with dir. Local and halo + * candidates are ranked separately, only the halo one can be handed over. ---*/ struct CCandidate { unsigned long node = std::numeric_limits::max(); unsigned long halo = std::numeric_limits::max(); @@ -2085,14 +1972,13 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (len <= 0.0) continue; for (unsigned short d = 0; d < nDim; ++d) vec[d] /= len; - /*--- The direction only RANKS candidates; a front is never stopped for turning. ---*/ const su2double dot = GeometryToolbox::DotProduct(nDim, vec, marchDir); const bool admissible = !(onPhysicalBoundary[jPoint] && EntersBoundary(fine_grid, config, nDim, jPoint, vec, cos_boundary)) && GeometricalCheck(jPoint, fine_grid, config); - /*--- Halo parents are assigned by the owning rank through the MPI relay, so claiming one here - * would fight that assignment. Admissibility is still checked, as the owner would. ---*/ + /*--- Halo parents are assigned by the owning rank through the MPI relay, so a halo node is + * only checked for admissibility here, never claimed. ---*/ if (!fine_grid->nodes->GetDomain(jPoint)) { if (dot > haloDot && admissible) { haloDot = dot; @@ -2112,14 +1998,13 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC return c; }; - /*--- Bid table: only an index per mesh point, the bids themselves in a compact vector. A whole - * CStep per point would be hundreds of megabytes that is empty almost everywhere. ---*/ + /*--- Bid table: an index per mesh point, the bids themselves in a compact vector. ---*/ constexpr unsigned NOBID = std::numeric_limits::max(); vector bidIdx(nPointFine, NOBID); vector bids; vector bidOwner; - /*--- Scratch for the layer under construction, hoisted so a front does not allocate per layer. ---*/ + /*--- Scratch for the layer under construction. ---*/ vector newLayer; /*--- Summed over all ranks for the one-line report at the end. ---*/ @@ -2135,7 +2020,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC vector inherited; /*--- Where each halo node sits in this rank's SEND_RECEIVE receive lists, so a handover can be - * packed against the right vertex without searching. A halo node appears in exactly one. ---*/ + * packed against the right vertex. ---*/ vector haloMarker(nPointFine, -1); vector haloVertex(nPointFine, 0); for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { @@ -2151,9 +2036,6 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC auto markFail = [&](unsigned long f) { fronts[f].failed = 1; }; - /*--- How many fine layers the next coarse CV of this front holds: two, so the stack coarsens by the - * same factor along the marching direction as the footprint does across it. One only if a second - * layer would exceed the agglomeration size limit. ---*/ /*--- Turn everything buffered for this front into one coarse control volume. ---*/ auto emit = [&](unsigned long f) { if (fronts[f].pending.empty()) return; @@ -2172,12 +2054,10 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC fronts[f].nBlock = BlockFor(maxAgglomSize, fronts[f].nodes); }; - /*--- The boundary layer of every front. Claiming it before ordinary boundary agglomeration runs is - * what keeps the stack aligned: every layer above has exactly the same footprint. ---*/ + /*--- The boundary layer of every front, claimed before ordinary boundary agglomeration runs so + * that every layer above keeps the same footprint. ---*/ for (const auto& patch : patches) { - /*--- A one-node patch may seed a front and marches as a stack one node wide. Leaving it to - * ordinary agglomeration is worse: its whole column then goes unpaved, and such seeds sit where - * markers meet, cutting full-height stripes through the paved region. ---*/ + /*--- A one-node patch may seed a front and marches as a stack one node wide. ---*/ bool valid = !patch.empty(); for (auto si : patch) { const auto p = seeds.node[si]; @@ -2198,9 +2078,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (nrm <= 0.0) continue; for (unsigned short d = 0; d < nDim; ++d) n0[d] /= nrm; - /*--- The boundary layer is a coarse CV on its own: a boundary node is never merged with an - * interior one, so only the first advance is a single layer. This also isolates a junction - * node without a rule of its own, since such a node is a patch of one. ---*/ + /*--- The boundary layer is a coarse CV on its own, so only the first advance is a single + * layer. ---*/ const auto f = addFront(layer0, n0, frontTag + 1, 1); for (auto p : layer0) { claimed[p] = 1; @@ -2211,8 +2090,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } for (unsigned long layer = 1;; ++layer) { - /*--- Every rank runs the same number of rounds: each ends in a collective handover exchange, so - * one dropping out early would hang the others. ---*/ + /*--- Every rank runs the same number of rounds, each ends in a collective handover + * exchange. ---*/ int aliveLocal = 0; for (unsigned long f = 0; f < fronts.size(); ++f) aliveLocal |= fronts[f].alive; int aliveGlobal = 0; @@ -2224,9 +2103,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC bids.clear(); bidOwner.clear(); - /*--- (a) Every alive front proposes a successor for each of its nodes. A front that cannot fill a - * whole layer proposes NOTHING: it is retiring this round anyway, and letting its partial bids - * stand would let a dying front displace a healthy one out of nodes it can still use. ---*/ + /*--- (a) Every alive front proposes a successor for each of its nodes. A front that cannot fill + * a whole layer proposes nothing and retires this round. ---*/ for (unsigned long f = 0; f < fronts.size(); ++f) { if (!fronts[f].alive) continue; fronts[f].prop.clear(); @@ -2235,8 +2113,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC for (auto n : fronts[f].nodes) { const auto c = bestSuccessor(n, fronts[f].dir.data()); - /*--- Nothing free here but the stack continues across the interface; the split test below - * decides whether the whole layer goes over. ---*/ + /*--- Nothing free here but the stack continues across the interface. ---*/ if ((c.node == NO_POINT) && (c.halo != NO_POINT)) { fronts[f].handTo.push_back(c.halo); continue; @@ -2252,21 +2129,19 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC fronts[f].prop.push_back(s); } - /*--- An interface can cut a footprint. If all of it crosses, the stack is handed over intact - * and this front is finished. If only part crosses, the footprint is SPLIT: the piece whose - * successors are local marches on here, the rest is handed across, and both are renamed. ---*/ + /*--- An interface can cut a footprint. All of it crossing hands the stack over intact, part of + * it crossing splits the footprint into a piece that stays and a piece handed across. ---*/ if (fronts[f].failed) { fronts[f].prop.clear(); fronts[f].handTo.clear(); } else if (!fronts[f].handTo.empty()) { - /*--- fronts[f].prop is built in the order of fronts[f].nodes, so this is the piece that stays, in the same - * order, and phi still runs index for index between it and the layer it proposes. ---*/ + /*--- fronts[f].prop is built in the order of fronts[f].nodes, so this is the piece that + * stays, in the same order as the layer it proposes. ---*/ vector narrow; for (const auto& s : fronts[f].prop) narrow.push_back(s.from); - /*--- A cut can leave the local piece in two disconnected halves - a square footprint cut - * diagonally does exactly that - and that is not a layer. Drop it and hand over the rest; - * the stack still survives on the far side instead of ending here. ---*/ + /*--- A cut can leave the local piece disconnected, which is not a layer. Drop it and hand + * over the rest. ---*/ if (!narrow.empty() && !IsConnectedLayer(fine_grid, narrow)) { narrow.clear(); } @@ -2276,8 +2151,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (narrow.empty()) { fronts[f].prop.clear(); } else { - /*--- Close the coarse CV that is open on the WIDE footprint before narrowing, so that no CV - * ever ends up holding two layers of different shape. ---*/ + /*--- Close the coarse CV open on the wide footprint before narrowing, so that no CV holds + * two layers of different shape. ---*/ fronts[f].nodes = narrow; emit(f); fronts[f].nBlock = BlockFor(maxAgglomSize, fronts[f].nodes); @@ -2288,8 +2163,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } /*--- (b) Contention resolved from bids that were all collected before any was granted, so the - * outcome is a pure function of the proposals and does not depend on the order the fronts are - * visited in. That is what makes the coarse grid reproducible. ---*/ + * outcome does not depend on the order the fronts are visited in. ---*/ auto better = [](const CStep& a, const CStep& b) { if (a.score != b.score) return a.score > b.score; if (a.dist != b.dist) return a.dist < b.dist; @@ -2299,8 +2173,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC for (unsigned long f = 0; f < fronts.size(); ++f) { if (!fronts[f].alive || fronts[f].failed) continue; for (const auto& s : fronts[f].prop) { - /*--- A front that has lost a bid is retiring and must not displace a healthy one with the - * rest of its layer. What it placed before losing stays, which is conservative. ---*/ + /*--- A front that has lost a bid retires and places no more of its layer. ---*/ if (fronts[f].failed) break; if (bidIdx[s.node] == NOBID) { @@ -2312,8 +2185,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC const auto k = bidIdx[s.node]; const auto g = bidOwner[k]; - /*--- Two nodes of the SAME front reaching for one successor is a pinch: the layer would come - * out narrower than the front, which all-or-nothing does not allow. ---*/ + /*--- Two nodes of the same front reaching for one successor is a pinch, the layer would come + * out narrower than the front. ---*/ if (better(s, bids[k])) { markFail(g); bids[k] = s; @@ -2322,8 +2195,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC markFail(f); } - /*--- A head-on meeting stops BOTH fronts, or the winner overshoots into the other's - * territory. A glancing contact is not a meeting and only costs the loser. ---*/ + /*--- A head-on meeting stops both fronts, a glancing contact only costs the loser. ---*/ if ((g != f) && (GeometryToolbox::DotProduct(nDim, fronts[f].dir.data(), fronts[g].dir.data()) < 0.0)) { markFail(f); markFail(g); @@ -2332,14 +2204,13 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } /*--- (c) All-or-nothing acceptance: a front takes the whole layer or none of it and retires. A - * bid only becomes a claim here, so a retiring front never has to give anything back and the - * nodes it was reaching for stay available to ordinary agglomeration. ---*/ + * bid only becomes a claim here. ---*/ for (unsigned long f = 0; f < fronts.size(); ++f) { if (!fronts[f].alive) continue; newLayer.clear(); if (!fronts[f].failed) { - /*--- Built in proposal order, so newLayer[k] is the successor of nodes[k] and the two carry phi. ---*/ + /*--- Built in proposal order, so newLayer[k] is the successor of nodes[k]. ---*/ for (const auto& s : fronts[f].prop) { const auto k = bidIdx[s.node]; if ((k != NOBID) && (bidOwner[k] == f)) newLayer.push_back(s.node); @@ -2350,7 +2221,6 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } if (fronts[f].failed) { - /*--- Nothing to give back: a bid only becomes a claim on acceptance. ---*/ fronts[f].alive = 0; emit(f); continue; @@ -2382,11 +2252,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (fronts[f].pendingLayers >= fronts[f].nBlock) emit(f); } - /*--- (d) Hand stacks across partition interfaces. A front that runs into the halo cannot go on - * here, so the footprint is sent to the owning rank, which picks the stack up and carries on. - * What crosses is the footprint, not a coarse CV, so both halves keep the same shape. It travels - * the reverse of the usual halo direction: packed against the RECEIVE marker and sent to the - * rank that marker receives from. ---*/ + /*--- (d) Hand stacks across partition interfaces. The footprint is sent to the owning rank, + * packed against the receive marker and sent to the rank that marker receives from. ---*/ for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) { if (!((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0))) continue; @@ -2397,8 +2264,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC const auto nVertexS = fine_grid->nVertex[MarkerS]; const auto nVertexR = fine_grid->nVertex[MarkerR]; - /*--- Packed against the halo vertices, i.e. what this rank wants the neighbour to continue. - * Tag and direction go separately: the AD MPI wrapper has no byte type to send a struct. ---*/ + /*--- Packed against the halo vertices. Tag and direction go separately, the AD MPI wrapper has + * no byte type to send a struct. ---*/ vector tagOut(nVertexR, 0), tagIn(nVertexS, 0); vector dirOut(nVertexR * nDim, 0.0), dirIn(nVertexS * nDim, 0.0); @@ -2406,7 +2273,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC for (auto p : F.handTo) { if (haloMarker[p] != static_cast(MarkerR)) continue; const auto v = haloVertex[p]; - /*--- Two fronts reaching for one node: the lower tag takes it, the same way on both ranks. ---*/ + /*--- Two fronts reaching for one node: the lower tag takes it. ---*/ if ((tagOut[v] != 0) && (tagOut[v] <= F.handTag)) continue; tagOut[v] = F.handTag; for (unsigned short d = 0; d < nDim; ++d) dirOut[v * nDim + d] = F.dir[d]; @@ -2424,8 +2291,8 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } } - /*--- A front that handed its WHOLE footprint over is finished here; the neighbour owns the rest - * of the stack. One that handed over only a piece keeps marching on what was left of it. ---*/ + /*--- A front that handed its whole footprint over is finished here, one that handed over only a + * piece keeps marching on what was left. ---*/ for (unsigned long f = 0; f < fronts.size(); ++f) { if (fronts[f].handTo.empty()) continue; fronts[f].handTo.clear(); @@ -2435,7 +2302,7 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC } /*--- Adopt what the neighbours sent, tags ascending so arrival order cannot change the outcome. - * A footprint whose nodes are not all free is dropped and the stack simply ends. ---*/ + * A footprint whose nodes are not all free is dropped. ---*/ std::sort(inherited.begin(), inherited.end(), [](const CInherited& a, const CInherited& b) { return a.tag < b.tag; }); @@ -2450,14 +2317,13 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC if (claimed[p] || fine_grid->nodes->GetAgglomerate(p) || !GeometricalCheck(p, fine_grid, config)) ok = false; layer0.push_back(p); } - /*--- The footprint has to arrive whole and connected, the same test any other layer passes. ---*/ + /*--- The footprint has to arrive whole and connected. ---*/ if (ok && !IsConnectedLayer(fine_grid, layer0)) ok = false; if (ok) { std::array d0{}; for (unsigned short d = 0; d < nDim; ++d) d0[d] = inherited[i].dir[d]; - /*--- An inherited layer is an interior one, so it is NOT subject to the single-layer rule the - * boundary layer gets: it opens an ordinary two-deep coarse CV and waits for its partner. ---*/ + /*--- An inherited layer is an interior one, so it opens an ordinary two-deep coarse CV. ---*/ const auto nf = addFront(layer0, d0, inherited[i].tag, BlockFor(maxAgglomSize, layer0)); for (auto p : layer0) { claimed[p] = 1; @@ -2470,11 +2336,10 @@ string CMultiGridGeometry::AgglomerateImplicitLines(unsigned long& Index_CoarseC inherited.clear(); } - /*--- Nothing should be left buffered, but a front retired outside the loop would strand its nodes - * with a parent index that was never assigned. ---*/ + /*--- Emit whatever is still buffered, so no node is left without a parent index. ---*/ for (unsigned long f = 0; f < fronts.size(); ++f) emit(f); - /*--- A rank with no fronts leaves dmin at its sentinel so it stays out of the MPI_MIN. ---*/ + /*--- A rank with no fronts leaves dmin at its sentinel, keeping it out of the MPI_MIN. ---*/ unsigned long dmin = std::numeric_limits::max(), dmax = 0; for (unsigned long f = 0; f < fronts.size(); ++f) { if (fronts[f].nodes.empty()) continue; diff --git a/TestCases/rans/rae2822/turb_SA_RAE2822.cfg b/TestCases/rans/rae2822/turb_SA_RAE2822.cfg index 2157904ca34..a64e5c04a8e 100644 --- a/TestCases/rans/rae2822/turb_SA_RAE2822.cfg +++ b/TestCases/rans/rae2822/turb_SA_RAE2822.cfg @@ -41,9 +41,9 @@ MARKER_MONITORING= ( AIRFOIL ) % ------------- COMMON PARAMETERS DEFINING THE NUMERICAL METHOD ---------------% % NUM_METHOD_GRAD= WEIGHTED_LEAST_SQUARES -CFL_NUMBER= 10.0 +CFL_NUMBER= 1000.0 CFL_ADAPT= NO -CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) +CFL_ADAPT_PARAM= ( 0.9, 1.1, 1.0, 10000.0, 1.00 ) ITER= 99999 LINEAR_SOLVER= BCGSTAB LINEAR_SOLVER_ERROR= 1E-1 @@ -51,13 +51,14 @@ LINEAR_SOLVER_ITER= 3 % -------------------------- MULTIGRID PARAMETERS -----------------------------% % -MGLEVEL= 3 +MGLEVEL= 2 MGCYCLE= W_CYCLE -MG_PRE_SMOOTH= ( 4, 4, 4, 4 ) -MG_POST_SMOOTH= ( 4, 4, 4, 4 ) +MG_PRE_SMOOTH= ( 1, 2, 2, 2 ) +MG_POST_SMOOTH= ( 2, 2, 2, 2 ) MG_CORRECTION_SMOOTH= ( 1, 1, 1, 1 ) -MG_DAMP_RESTRICTION= 0.5 -MG_DAMP_PROLONGATION= 0.5 +MG_DAMP_RESTRICTION= 0.75 +MG_DAMP_PROLONGATION= 0.75 +MG_IMPLICIT_LINES= YES % -------------------- FLOW NUMERICAL METHOD DEFINITION -----------------------% % @@ -104,8 +105,9 @@ GRAD_OBJFUNC_FILENAME= of_grad SURFACE_FILENAME= surface_flow SURFACE_ADJ_FILENAME= surface_adjoint OUTPUT_WRT_FREQ= 250 -SCREEN_OUTPUT= (INNER_ITER, RMS_DENSITY, RMS_NU_TILDE, LIFT, DRAG, TOTAL_HEATFLUX) - +SCREEN_OUTPUT= (INNER_ITER, RMS_DENSITY, RMS_NU_TILDE, LIFT, DRAG, AVG_CFL) +VOLUME_OUTPUT=SOLUTIOn,PRIMITIVE,RESIDUAL,RANK,MULTIGRID +WRT_PERFORMANCE= YES % ------------------------ GRID DEFORMATION PARAMETERS ------------------------% % DV_KIND= FFD_SETTING