From ce94d1f7fee7a7c98edf3e68b414072dc4ad7984 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 18 Aug 2025 16:45:27 -0300 Subject: [PATCH 01/13] improve cost calculation --- algorithms/partitioner/communication_cost.cpp | 118 +++++++++--------- algorithms/partitioner/communication_cost.hpp | 19 +-- 2 files changed, 62 insertions(+), 75 deletions(-) diff --git a/algorithms/partitioner/communication_cost.cpp b/algorithms/partitioner/communication_cost.cpp index 93dcb7b2..644c3f4f 100644 --- a/algorithms/partitioner/communication_cost.cpp +++ b/algorithms/partitioner/communication_cost.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with SBG Library. If not, see . - ******************************************************************************/ +******************************************************************************/ #include @@ -36,41 +36,21 @@ static CommunicationCost* cost_matrix = nullptr; namespace { -ec_ic compute_EC_IC_from_map_1_to_map_2( - const Partition& partition, - const SetPiece& nodes, - const PWMap& map_1, - const PWMap& map_2, - const SetAF& set_fact) +Set set_piece_communication(const SetPiece& nodes, const WeightedSBGraph& graph) { - auto nodes_set = set_fact.createSet(nodes); - auto d = map_1.preImage(nodes_set); - auto im = map_2.image(d); - auto partition_set = from_vector(partition, set_fact); - auto ic_nodes = partition_set.intersection(im); - ic_nodes = ic_nodes.difference(nodes_set); - auto ec_nodes = im.difference(ic_nodes); - auto ic = map_2.preImage(ic_nodes).intersection(d); - auto ec = map_2.preImage(ec_nodes).intersection(d); - - return make_pair(ec, ic); -} - - -} + // convert nodes into a set + auto node_set = graph.fact().createSet(nodes); + // compute preImage of map1 and map2 to get the edges that connects `nodes` + auto edges_map1 = graph.map1().preImage(node_set); + auto edges_map2 = graph.map2().preImage(node_set); -ec_ic compute_EC_IC( - const Partition& partition, - const SetPiece& nodes, - const SBG::LIB::WeightedSBGraph& graph) -{ - ec_ic cost1 = compute_EC_IC_from_map_1_to_map_2(partition, nodes, graph.map1(), graph.map2(), graph.fact()); - ec_ic cost2 = compute_EC_IC_from_map_1_to_map_2(partition, nodes, graph.map2(), graph.map1(), graph.fact()); + // Now compute the disjoint union to remove loop edges + auto communication = edges_map1.cup(edges_map2).difference(edges_map1.intersection(edges_map2)); - ec_ic cost = ec_ic(cost1.first.cup(cost2.first), cost1.second.cup(cost2.second)); + return communication; +} - return cost; } } @@ -91,17 +71,21 @@ void CommunicationCost::initialize() _ec_cost_by_interval.reserve(_partitions.size()); _ic_cost_by_interval.reserve(_partitions.size()); for (size_t i = 0; i < _partitions.size(); i++) { + Set partition_i_communication = _graph.fact().createSet(); + Set internal_communication_partition_i = _graph.fact().createSet(); - _cost_by_partition.emplace_back(make_pair(_graph.fact().createSet(), _graph.fact().createSet())); - _ec_cost_by_interval.emplace_back(); - _ic_cost_by_interval.emplace_back(); for (const auto& node : _partitions.at(i)) { - auto [ec, ic] = internal::compute_EC_IC(_partitions.at(i), node, _graph); - - _cost_by_partition.back() = { _cost_by_partition.back().first.cup(ec), _cost_by_partition.back().second.cup(ic) }; - _ec_cost_by_interval.back().insert({node, ec}); - _ic_cost_by_interval.back().insert({node, ic}); + auto node_edges = internal::set_piece_communication(node, _graph); + _communication_by_set_piece.insert({node, node_edges}); + internal_communication_partition_i = node_edges.intersection(partition_i_communication).cup(internal_communication_partition_i); + partition_i_communication = partition_i_communication.cup(node_edges); } + + auto ec_parition_i = partition_i_communication.difference(internal_communication_partition_i); + _cost_by_partition.emplace_back(make_pair(ec_parition_i, move(internal_communication_partition_i))); + + _ic_cost_by_interval.emplace_back(); // save space for this, will be filled on demand + _ec_cost_by_interval.emplace_back(); } } @@ -110,21 +94,27 @@ void CommunicationCost::update_partitions(PartitionMap& partitions, optional; - -/** - * @brief This function computes external and internal communication - * given a certain partition for a particular set piece. - * - * @param partition - partition of graph nodes. - * @param nodes - set piece of the graph nodes. - * @param graph - set based graph that is being partitioned. */ -ec_ic compute_EC_IC( - const Partition& partition, - const SBG::LIB::SetPiece& nodes, - const SBG::LIB::WeightedSBGraph& graph); - -} - class CommunicationCost { public: CommunicationCost(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); @@ -97,6 +79,7 @@ class CommunicationCost { const SBG::LIB::WeightedSBGraph& _graph; // read-only members PartitionMap _partitions; + std::unordered_map _communication_by_set_piece; std::vector> _cost_by_partition; std::vector> _ec_cost_by_interval; std::vector> _ic_cost_by_interval; From c809b4d702fee56ae88a1b809380969826fa2305 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Wed, 20 Aug 2025 13:53:34 -0300 Subject: [PATCH 02/13] improve some data structures --- algorithms/partitioner/build_sb_graph.cpp | 12 ++++----- algorithms/partitioner/communication_cost.cpp | 4 +-- algorithms/partitioner/communication_cost.hpp | 3 ++- .../partitioner/kernighan_lin_partitioner.cpp | 27 +++++-------------- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/algorithms/partitioner/build_sb_graph.cpp b/algorithms/partitioner/build_sb_graph.cpp index 19c90320..0335630d 100644 --- a/algorithms/partitioner/build_sb_graph.cpp +++ b/algorithms/partitioner/build_sb_graph.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,7 @@ namespace { struct Var { string id; vector> exps; - vector defs; + list defs; unsigned cost = 1; }; @@ -115,7 +116,7 @@ struct Node { vector read_var_object(const rapidjson::Value& var_array) { vector vars; - // vars.reserve(var_array.GetArray().Size()); + vars.reserve(var_array.GetArray().Size()); for (const auto& value : var_array.GetArray()) { assert(value.HasMember("id") and value["id"].IsString()); string id = value["id"].GetString(); @@ -134,12 +135,11 @@ vector read_var_object(const rapidjson::Value& var_array) int exp_a = expression[0].GetInt(); int exp_b = expression[1].GetInt(); - exps.push_back(make_pair(exp_a, exp_b)); + exps.emplace_back(exp_a, exp_b); } auto def_object = value["defs"].GetArray(); - vector defs; - defs.reserve(def_object.Size()); + list defs; for (const auto& def : def_object) { defs.push_back(def.GetInt()); } @@ -222,7 +222,7 @@ tuple create_set_of_nodes(const map& nodes, map> modified_partitions) +void CommunicationCost::update_partitions(PartitionMap& partitions, optional>> modified_partitions) { _partitions = partitions; if (modified_partitions) { // now, update communication for partitions that were updated - for (size_t i : *modified_partitions) { + for (size_t i : modified_partitions->get()) { Set partition_i_communication = _graph.fact().createSet(); Set internal_communication_partition_i = _graph.fact().createSet(); diff --git a/algorithms/partitioner/communication_cost.hpp b/algorithms/partitioner/communication_cost.hpp index 505c84fa..dcad45fa 100644 --- a/algorithms/partitioner/communication_cost.hpp +++ b/algorithms/partitioner/communication_cost.hpp @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -40,7 +41,7 @@ class CommunicationCost { * @param partitions - new partition of the graph nodes. * @param modified_partitions - [optional] partitions that were updated. */ - void update_partitions(PartitionMap& partitions, std::optional> modified_partitions = std::nullopt); + void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); /** * It returns the edges that communicate nodes in partition `partition_id` with others. diff --git a/algorithms/partitioner/kernighan_lin_partitioner.cpp b/algorithms/partitioner/kernighan_lin_partitioner.cpp index a0309617..1929399a 100644 --- a/algorithms/partitioner/kernighan_lin_partitioner.cpp +++ b/algorithms/partitioner/kernighan_lin_partitioner.cpp @@ -286,22 +286,6 @@ GainObjectImbalance update_diff(CostMatrixImbalance& cost_matrix, Partition& rem return max_gain_object; } -// auto return type we’ll let the compiler deduce what the return type should be from the return statement -auto max_diff(CostMatrixImbalance& cost_matrix) -{ - // cost_matrix is sort by gain, so the first is the maximum gain - auto g = cost_matrix.begin(); - - auto gain_object = *g; -#if PARTITION_IMBALANCE_DEBUG - logging::sbg_log << "The best is " << *g << endl; -#endif - - // remove it, we need to update those values that - cost_matrix.erase(g); - - return gain_object; -} void update_sum(int& par_sum, int g, int& max_par_sum, pair& max_par_sum_set, const Set& a_v, const Set& b_v) { @@ -312,6 +296,7 @@ void update_sum(int& par_sum, int g, int& max_par_sum, pair& max_par_s } } + int kl_sbg_imbalance(const WeightedSBGraph& graph, CommunicationCost& cost_matrix, unsigned partition_a_id, Partition& partition_a, unsigned partition_b_id, Partition& partition_b, unsigned LMin, unsigned LMax) { @@ -386,7 +371,7 @@ KLBipartResult kl_sbg_bipart_imbalance(const WeightedSBGraph& graph, Communicati } KLSbgPartitionerResult kl_sbg_partitioner_function(const WeightedSBGraph& graph, PartitionMap& partitions, CommunicationCost& cost_matrix, - unsigned LMin, unsigned LMax, vector& gains) + unsigned LMin, unsigned LMax, list& gains) { KLSbgPartitionerResult best_gain = KLSbgPartitionerResult{0, 0, -1, {}, {}}; for (size_t i = 0; i < partitions.size(); i++) { @@ -426,9 +411,9 @@ KLSbgPartitionerResult kl_sbg_partitioner_function(const WeightedSBGraph& graph, KLSbgPartitionerResult kl_sbg_partitioner_multithreading(const WeightedSBGraph& graph, PartitionMap& partitions, CommunicationCost& cost_matrix, unsigned LMin, unsigned LMax, - vector& gains) + list& gains) { - vector> workers; + list> workers; KLSbgPartitionerResult best_gain = KLSbgPartitionerResult{0, 0, -1, {}, {}}; for (size_t i = 0; i < partitions.size(); i++) { const auto ec_partition_i = cost_matrix.get_ec_by_partition_id(i); @@ -605,7 +590,7 @@ void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& pa int counter = 0; CommunicationCost& cost_matrix = get_communication_cost(); - vector gains; + list gains; while (change) { cout << "*****ITERATION NUMBER " << counter++ << endl; change = false; @@ -640,7 +625,7 @@ void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& pa default: int it_counter = 0; - vector modified_partitions = {}; + list modified_partitions = {}; while (not gains.empty() and best_gain.gain > 0) { logging::sbg_log << "change number " << it_counter << " changing " << best_gain.i << ", " << best_gain.j << endl; it_counter++; From afe31b8efdd57c37325575d1197db32d38953a4a Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Wed, 20 Aug 2025 15:17:44 -0300 Subject: [PATCH 03/13] avoid code repetition --- algorithms/partitioner/communication_cost.cpp | 31 ++++++++++++------- algorithms/partitioner/communication_cost.hpp | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/algorithms/partitioner/communication_cost.cpp b/algorithms/partitioner/communication_cost.cpp index 388be0a1..55365b49 100644 --- a/algorithms/partitioner/communication_cost.cpp +++ b/algorithms/partitioner/communication_cost.cpp @@ -131,12 +131,8 @@ Set CommunicationCost::get_ec_by_partition_id(unsigned partition_id) } -Set CommunicationCost::get_ec_by_interval(unsigned partition_id, const SetPiece& nodes) +pair CommunicationCost::compute_ec_ic(unsigned partition_id, const SetPiece& nodes) { - if (_ec_cost_by_interval[partition_id].find(nodes) != _ec_cost_by_interval[partition_id].end()) { - return _ec_cost_by_interval[partition_id].at(nodes); - } - if (_communication_by_set_piece.find(nodes) == _communication_by_set_piece.end()) { _communication_by_set_piece.insert({nodes, internal::set_piece_communication(nodes, _graph)}); } @@ -144,10 +140,26 @@ Set CommunicationCost::get_ec_by_interval(unsigned partition_id, const SetPiece& auto communication = _communication_by_set_piece.at(nodes); auto ec = communication.intersection(_cost_by_partition[partition_id].first); - auto ic = communication.intersection(_cost_by_partition[partition_id].second); + auto ic = communication.difference(ec); _ec_cost_by_interval[partition_id].insert({nodes, ec}); _ic_cost_by_interval[partition_id].insert({nodes, ic}); + return { ec, ic }; +} + + +Set CommunicationCost::get_ec_by_interval(unsigned partition_id, const SetPiece& nodes) +{ + if (_ec_cost_by_interval[partition_id].find(nodes) != _ec_cost_by_interval[partition_id].end()) { + return _ec_cost_by_interval[partition_id].at(nodes); + } + + if (_communication_by_set_piece.find(nodes) == _communication_by_set_piece.end()) { + _communication_by_set_piece.insert({nodes, internal::set_piece_communication(nodes, _graph)}); + } + + auto [ec, _] = compute_ec_ic(partition_id, nodes); + return ec; } @@ -162,12 +174,7 @@ Set CommunicationCost::get_ic_by_interval(unsigned partition_id, const SetPiece& _communication_by_set_piece.insert({nodes, internal::set_piece_communication(nodes, _graph)}); } - auto communication = _communication_by_set_piece.at(nodes); - - auto ec = communication.intersection(_cost_by_partition[partition_id].first); - auto ic = communication.intersection(_cost_by_partition[partition_id].second); - _ec_cost_by_interval[partition_id].insert({nodes, ec}); - _ic_cost_by_interval[partition_id].insert({nodes, ic}); + auto [_, ic] = compute_ec_ic(partition_id, nodes); return ic; } diff --git a/algorithms/partitioner/communication_cost.hpp b/algorithms/partitioner/communication_cost.hpp index dcad45fa..9e727f78 100644 --- a/algorithms/partitioner/communication_cost.hpp +++ b/algorithms/partitioner/communication_cost.hpp @@ -86,7 +86,7 @@ class CommunicationCost { std::vector> _ic_cost_by_interval; void initialize(); - + std::pair compute_ec_ic(unsigned partition_id, const SBG::LIB::SetPiece& nodes); }; /** From d964af01d5725f7e40a40c10baf5608a309566fd Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 25 Aug 2025 13:32:52 -0300 Subject: [PATCH 04/13] select if using multiple intial paritition strategies by a flag --- algorithms/partitioner/dfs_on_sbg.cpp | 8 +- algorithms/partitioner/dfs_on_sbg.hpp | 8 +- algorithms/partitioner/main.cpp | 355 +++++++++--------- algorithms/partitioner/partition_graph.cpp | 92 +++-- algorithms/partitioner/partition_graph.hpp | 20 +- algorithms/partitioner/partition_strategy.hpp | 2 +- test/partitioner/partitioner_test.cpp | 4 +- 7 files changed, 260 insertions(+), 229 deletions(-) diff --git a/algorithms/partitioner/dfs_on_sbg.cpp b/algorithms/partitioner/dfs_on_sbg.cpp index d9f2dadb..50c04606 100644 --- a/algorithms/partitioner/dfs_on_sbg.cpp +++ b/algorithms/partitioner/dfs_on_sbg.cpp @@ -41,7 +41,7 @@ void initialize_partitioning(SBG::LIB::WeightedSBGraph& graph, unsigned number_o sort_object = DFS(graph, number_of_partitions); } -void add_strategy(PartitionStrategy& strategy, bool pre_order) { sort_object.add_partition_strategy(strategy, pre_order); } +void add_strategy(unique_ptr&& strategy, bool pre_order) { sort_object.add_partition_strategy(move(strategy), pre_order); } vector>> partitionate() { @@ -167,12 +167,12 @@ void DFS::iterate() } } -void DFS::add_partition_strategy(PartitionStrategy& strategy, bool pre_order) +void DFS::add_partition_strategy(unique_ptr&& strategy, bool pre_order) { if (pre_order) { - _partition_strategy_pre_order.push_back(&strategy); + _partition_strategy_pre_order.push_back(move(strategy)); } else { - _partition_strategy_post_order.push_back(&strategy); + _partition_strategy_post_order.push_back(move(strategy)); } } diff --git a/algorithms/partitioner/dfs_on_sbg.hpp b/algorithms/partitioner/dfs_on_sbg.hpp index 5b7d6f30..28e008f3 100644 --- a/algorithms/partitioner/dfs_on_sbg.hpp +++ b/algorithms/partitioner/dfs_on_sbg.hpp @@ -37,7 +37,7 @@ namespace search { void initialize_partitioning(SBG::LIB::WeightedSBGraph& graph, unsigned number_of_partitions); -void add_strategy(PartitionStrategy& strategy, bool pre_order); +void add_strategy(std::unique_ptr&& strategy, bool pre_order); std::vector>> partitionate(); @@ -63,7 +63,7 @@ class DFS { std::vector>> partitions() const; /// @note strategy object should live while this class does - void add_partition_strategy(PartitionStrategy& strategy, bool pre_order); + void add_partition_strategy(std::unique_ptr&& strategy, bool pre_order); private: typedef SBG::LIB::Set::Iterator node_identifier; @@ -81,8 +81,8 @@ class DFS { SBG::LIB::WeightedSBGraph* _graph; SBG::LIB::Set _nodes; - std::vector _partition_strategy_pre_order; - std::vector _partition_strategy_post_order; + std::vector> _partition_strategy_pre_order; + std::vector> _partition_strategy_post_order; void initialize_adjacents(); diff --git a/algorithms/partitioner/main.cpp b/algorithms/partitioner/main.cpp index 4c980e1c..2e7e626b 100644 --- a/algorithms/partitioner/main.cpp +++ b/algorithms/partitioner/main.cpp @@ -38,222 +38,239 @@ using namespace sbg_partitioner; static void usage() { - cout << "Usage sbg-partitioner" << endl; - cout << endl; - cout << "-f, --filename Path to the input file, a json file that represents " + cout << "Usage sbg-partitioner" << endl; + cout << endl; + cout << "-f, --filename Path to the input file, a json file that represents " "the model we want to partitionate." - << endl; - cout << "-p, --partitions Number of partitions." << endl; - cout << "-h, --help Display this information and exit." << endl; - cout << "-v, --version Display version information and exit." << endl; - cout << "-g Output file path." << endl; - cout << "-e Imbalance epsilon, a value between 0 and 1." << endl; - cout << "-m, --compute-metrics If enabled, computes partition quality metrics." << endl; - cout << "-d, --directory Directory with partitions obtianed by other partitioners, " + << endl; + cout << "-p, --partitions Number of partitions." << endl; + cout << "-h, --help Display this information and exit." << endl; + cout << "-v, --version Display version information and exit." << endl; + cout << "-g Output file path." << endl; + cout << "-e Imbalance epsilon, a value between 0 and 1." << endl; + cout << "-m, --compute-metrics If enabled, computes partition quality metrics." << endl; + cout << "-d, --directory Directory with partitions obtianed by other partitioners, " "we want to run quality metrics against them." - << endl; - cout << endl; - cout << "SBG Partitioner home page: https://github.com/CIFASIS/sbg-partitioner " << endl; + << endl; + cout << "-i, --initial-partition-strategy Choose a particular initial partition strategy. If this " + "flag is disbaled, all strategies will be computed and the best partition will be chosen.\n" + "\tValue\tSearching algorithm\tStrategy\tOrder\n" + "\t0\tDepth first search\tDistributive\tpreorder\n" + "\t1\tDepth first search\tDistributive\tpostorder\n" + "\t2\tDepth first search\tGreedy\t\tpreorder\n" + "\t3\tDepth first search\tGreedy\t\tpostorder\n" + << endl; + + cout << endl; + cout << "SBG Partitioner home page: https://github.com/CIFASIS/sbg-partitioner " << endl; } static void version() { - cout << "SBG Partitioner 1.0.0" << endl; - cout << "License GPLv3+: GNU GPL version 3 or later " << endl; - cout << "This is free software: you are free to change and redistribute it." << endl; - cout << "There is NO WARRANTY, to the extent permitted by law." << endl; + cout << "SBG Partitioner 1.0.0" << endl; + cout << "License GPLv3+: GNU GPL version 3 or later " << endl; + cout << "This is free software: you are free to change and redistribute it." << endl; + cout << "There is NO WARRANTY, to the extent permitted by law." << endl; } void sort_before_print(PartitionMap partitions, const SBG::LIB::WeightedSBGraph& sb_graph, SBG::LIB::SetAF& set_fact) { - for (auto& p : partitions) { - sort_partition_intervals(p); - } - cout << "partitions: " << partitions << endl; + for (auto& p : partitions) { + sort_partition_intervals(p); + } + cout << "partitions: " << partitions << endl; } void read_directory(const std::string& name, std::vector& v) { - auto path_leaf_string = [](const std::filesystem::directory_entry& entry) { return entry.path().string(); }; + auto path_leaf_string = [](const std::filesystem::directory_entry& entry) { return entry.path().string(); }; - std::filesystem::path p(name); - std::filesystem::directory_iterator start(p); - std::filesystem::directory_iterator end; - std::transform(start, end, std::back_inserter(v), path_leaf_string); + std::filesystem::path p(name); + std::filesystem::directory_iterator start(p); + std::filesystem::directory_iterator end; + std::transform(start, end, std::back_inserter(v), path_leaf_string); } int main(int argc, char** argv) { - int opt; - optional filename = nullopt; - optional directory = nullopt; - optional number_of_partitions = nullopt; - optional output_file; - optional output_sb_graph = nullopt; - optional epsilon = nullopt; - bool compute_metrics = false; - - while (true) { - static struct option long_options[] = {{"filename", required_argument, 0, 'f'}, {"partitions", required_argument, 0, 'p'}, - {"output-file", required_argument, 0, 'g'}, {"output-graph", required_argument, 0, 'o'}, - {"compute-metrics", no_argument, 0, 'm'}, {"directory", required_argument, 0, 'd'}, - {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}}; - - int option_index = 0; - opt = getopt_long(argc, argv, "f:p:e:o:g:d:mvh:", long_options, &option_index); - if (opt == EOF) break; - - switch (opt) { - case 'f': - if (optarg) { - filename = string(optarg); - } - break; - - case 'p': - if (optarg) { - number_of_partitions = atoi(optarg); - } - break; - - case 'o': - if (optarg) { - output_sb_graph = string(optarg); - } - break; - - case 'g': - if (optarg) { - output_file = string(optarg); - } - break; - - case 'e': - if (optarg) { - epsilon = atof(optarg); - } - break; - - case 'm': - compute_metrics = true; - break; - - case 'd': - if (optarg) { - directory = string(optarg); - } - break; - - case 'v': - version(); - exit(0); - - case 'h': - usage(); - exit(0); - - case '?': - usage(); - exit(-1); - break; - - default: - cout << "opt " << opt << endl; - abort(); + int opt; + optional filename = nullopt; + optional directory = nullopt; + optional number_of_partitions = nullopt; + optional output_file; + optional output_sb_graph = nullopt; + optional epsilon = nullopt; + InitialPartitionStrategy initial_partition_strategy = InitialPartitionStrategy::ALL; + bool compute_metrics = false; + + while (true) { + static struct option long_options[] = {{"filename", required_argument, 0, 'f'}, {"partitions", required_argument, 0, 'p'}, + {"output-file", required_argument, 0, 'g'}, {"output-graph", required_argument, 0, 'o'}, + {"compute-metrics", no_argument, 0, 'm'}, {"directory", required_argument, 0, 'd'}, + {"initial-partition-strategy", required_argument, 0, 'i'}, + {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}}; + + int option_index = 0; + opt = getopt_long(argc, argv, "f:p:e:o:g:d:i:mvh:", long_options, &option_index); + if (opt == EOF) break; + + switch (opt) { + case 'f': + if (optarg) { + filename = string(optarg); + } + break; + + case 'p': + if (optarg) { + number_of_partitions = atoi(optarg); + } + break; + + case 'o': + if (optarg) { + output_sb_graph = string(optarg); + } + break; + + case 'g': + if (optarg) { + output_file = string(optarg); + } + break; + + case 'e': + if (optarg) { + epsilon = atof(optarg); + } + break; + + case 'm': + compute_metrics = true; + break; + + case 'd': + if (optarg) { + directory = string(optarg); + } + break; + + case 'i': + if (optarg) { + initial_partition_strategy = InitialPartitionStrategy(atoi(optarg)); + } + break; + + case 'v': + version(); + exit(0); + + case 'h': + usage(); + exit(0); + + case '?': + usage(); + exit(-1); + break; + + default: + cout << "opt " << opt << endl; + abort(); + } } - } - if (not filename or not number_of_partitions) { - usage(); - exit(1); - } + if (not filename or not number_of_partitions) { + usage(); + exit(1); + } - if (not epsilon) { - epsilon = 0.0; - } + if (not epsilon) { + epsilon = 0.0; + } - if (*epsilon < 0 or *epsilon > 1) { - usage(); - exit(1); - } + if (*epsilon < 0 or *epsilon > 1) { + usage(); + exit(1); + } - cout << "filename is " << *filename << endl; - cout << "number of partitions is " << *number_of_partitions << endl; + cout << "filename is " << *filename << endl; + cout << "number of partitions is " << *number_of_partitions << endl; - optional s; - if (output_sb_graph) { - s = ""; - } + optional s; + if (output_sb_graph) { + s = ""; + } - SBG::LIB::UnordAF set_fact; - SBG::LIB::MapAF map_fact(set_fact); - SBG::LIB::UnordPWMapAF pw_fact(map_fact); + SBG::LIB::UnordAF set_fact; + SBG::LIB::MapAF map_fact(set_fact); + SBG::LIB::UnordPWMapAF pw_fact(map_fact); - auto start_build_graph = chrono::high_resolution_clock::now(); - auto sb_graph = build_sb_graph(filename->c_str(), pw_fact); - auto end_build_graph = chrono::high_resolution_clock::now(); - auto time_to_build_graph = chrono::duration(end_build_graph - start_build_graph).count(); + auto start_build_graph = chrono::high_resolution_clock::now(); + auto sb_graph = build_sb_graph(filename->c_str(), pw_fact); + auto end_build_graph = chrono::high_resolution_clock::now(); + auto time_to_build_graph = chrono::duration(end_build_graph - start_build_graph).count(); - cout << "sb_graph: " << sb_graph << endl; + cout << "sb_graph: " << sb_graph << endl; - auto start_partitionate = chrono::high_resolution_clock::now(); - auto partitions = best_initial_partition(sb_graph, *number_of_partitions); - kl_sbg_imbalance_partitioner(sb_graph, partitions, *epsilon); - auto end_partitionate = chrono::high_resolution_clock::now(); - auto time_to_partitionate = chrono::duration(end_partitionate - start_partitionate).count(); + auto start_partitionate = chrono::high_resolution_clock::now(); + auto partitions = best_initial_partition(sb_graph, *number_of_partitions, initial_partition_strategy); + kl_sbg_imbalance_partitioner(sb_graph, partitions, *epsilon); + auto end_partitionate = chrono::high_resolution_clock::now(); + auto time_to_partitionate = chrono::duration(end_partitionate - start_partitionate).count(); - if (compute_metrics) { - map metrics; + if (compute_metrics) { + map metrics; - int edge_cut = metrics::edge_cut(partitions, sb_graph, set_fact); + int edge_cut = metrics::edge_cut(partitions, sb_graph, set_fact); - auto [comm_volume, max_comm_volume] = metrics::communication_volume(partitions, sb_graph, set_fact, map_fact); + auto [comm_volume, max_comm_volume] = metrics::communication_volume(partitions, sb_graph, set_fact, map_fact); - auto max_imb = metrics::maximum_imbalance(partitions, sb_graph, set_fact); + auto max_imb = metrics::maximum_imbalance(partitions, sb_graph, set_fact); - metrics::communication_metrics comm_metrics = metrics::communication_metrics{edge_cut, comm_volume, max_comm_volume, max_imb}; - metrics["sbg-partitioner"] = comm_metrics; + metrics::communication_metrics comm_metrics = metrics::communication_metrics{edge_cut, comm_volume, max_comm_volume, max_imb}; + metrics["sbg-partitioner"] = comm_metrics; - for (const auto& [f, m] : metrics) { - cout << f << ": " << m << endl; - } + for (const auto& [f, m] : metrics) { + cout << f << ": " << m << endl; + } - if (compute_metrics and directory) { - std::vector dir_files; - read_directory(*directory, dir_files); + if (compute_metrics and directory) { + std::vector dir_files; + read_directory(*directory, dir_files); - for (const auto& f : dir_files) { - auto partition_from_file = metrics::read_partition_from_file(f, sb_graph, set_fact); + for (const auto& f : dir_files) { + auto partition_from_file = metrics::read_partition_from_file(f, sb_graph, set_fact); - int edge_cut = metrics::edge_cut(partition_from_file, sb_graph, set_fact); + int edge_cut = metrics::edge_cut(partition_from_file, sb_graph, set_fact); - auto [comm_volume, max_comm_volume] = metrics::communication_volume(partition_from_file, sb_graph, set_fact, map_fact); + auto [comm_volume, max_comm_volume] = metrics::communication_volume(partition_from_file, sb_graph, set_fact, map_fact); - auto max_imb = metrics::maximum_imbalance(partition_from_file, sb_graph, set_fact); + auto max_imb = metrics::maximum_imbalance(partition_from_file, sb_graph, set_fact); - metrics::communication_metrics comm_metrics = metrics::communication_metrics{edge_cut, comm_volume, max_comm_volume, max_imb}; - metrics[std::filesystem::path(f).filename().string()] = comm_metrics; - } + metrics::communication_metrics comm_metrics = metrics::communication_metrics{edge_cut, comm_volume, max_comm_volume, max_imb}; + metrics[std::filesystem::path(f).filename().string()] = comm_metrics; + } + } } - } - // print profiler results if they are enabled - SBG::Util::time_profiler_results(); + // print profiler results if they are enabled + SBG::Util::time_profiler_results(); - cout << "time_to_build_graph = " << time_to_build_graph << " ms" << endl; - cout << "time_to_partitionate = " << time_to_partitionate << " ms" << endl; + cout << "time_to_build_graph = " << time_to_build_graph << " ms" << endl; + cout << "time_to_partitionate = " << time_to_partitionate << " ms" << endl; - if (sanity_check_enabled) { - sanity_check(sb_graph, partitions, *number_of_partitions); - } + if (sanity_check_enabled) { + sanity_check(sb_graph, partitions, *number_of_partitions); + } - if (s) { - s = get_pretty_sb_graph(sb_graph); - } + if (s) { + s = get_pretty_sb_graph(sb_graph); + } - sort_before_print(partitions, sb_graph, set_fact); + sort_before_print(partitions, sb_graph, set_fact); - string output = get_output(partitions); + string output = get_output(partitions); - return 0; + return 0; } diff --git a/algorithms/partitioner/partition_graph.cpp b/algorithms/partitioner/partition_graph.cpp index 7296f1a2..01b708da 100644 --- a/algorithms/partitioner/partition_graph.cpp +++ b/algorithms/partitioner/partition_graph.cpp @@ -30,7 +30,6 @@ #include "partition_graph.hpp" #include "sbg_partitioner_log.hpp" -#define TRY_MULTIPLE_STRATEGIES 1 using namespace std; @@ -68,48 +67,34 @@ Set get_communication_edges(Set partition, const PWMap& map_1, const PWMap& map_ return size; } -constexpr bool using_many_initial_partitions = TRY_MULTIPLE_STRATEGIES; -} // namespace - - -// we could cache solutions here -Set from_vector(const Partition& partition, const SetAF& set_fact) { - Set partition_set = set_fact.createSet(); - for (size_t i = 0; i < partition.size(); i++) { - partition_set.emplace(partition[i]); - } - - return partition_set; -} - - -Partition to_vector(const Set& partition_set) -{ - Partition partition; - for (auto set_piece : partition_set) { - partition.push_back(move(set_piece)); - } - - return partition; -} - -vector make_initial_partitions(SBG::LIB::WeightedSBGraph& graph, unsigned number_of_partitions) +vector make_initial_partitions(SBG::LIB::WeightedSBGraph& graph, unsigned number_of_partitions, + const InitialPartitionStrategy strategy) { vector partitions_sets; initialize_partitioning(graph, number_of_partitions); constexpr bool pre_order = true; - auto s1 = PartitionStrategyDistributive(number_of_partitions, graph); - add_strategy(s1, pre_order); -#if TRY_MULTIPLE_STRATEGIES - auto s2 = PartitionStrategyDistributive(number_of_partitions, graph); - add_strategy(s2, not pre_order); - auto s3 = PartitionStrategyGreedy(number_of_partitions, graph); - add_strategy(s3, pre_order); - auto s4 = PartitionStrategyGreedy(number_of_partitions, graph); - add_strategy(s4, not pre_order); -#endif + const bool all_strategies = strategy == InitialPartitionStrategy::ALL; + if (all_strategies or strategy == InitialPartitionStrategy::DFS_DISTRIBUTIVE_PREORDER) { + auto s1 = make_unique(number_of_partitions, graph); + add_strategy(move(s1), pre_order); + } + + if (all_strategies or strategy == InitialPartitionStrategy::DFS_DISTRIBUTIVE_POSTORDER) { + auto s2 = make_unique(number_of_partitions, graph); + add_strategy(move(s2), not pre_order); + } + + if (all_strategies or strategy == InitialPartitionStrategy::DFS_GREEDY_PREORDER) { + auto s3 = make_unique(number_of_partitions, graph); + add_strategy(move(s3), pre_order); + } + + if (all_strategies or strategy == InitialPartitionStrategy::DFS_GREEDY_POSTORDER) { + auto s4 = make_unique(number_of_partitions, graph); + add_strategy(move(s4), not pre_order); + } vector>> partitions = partitionate(); @@ -144,13 +129,40 @@ vector make_initial_partitions(SBG::LIB::WeightedSBGraph& graph, u return partitions_sets; } -PartitionMap best_initial_partition(WeightedSBGraph& graph, unsigned number_of_partitions) +} // namespace + + +// we could cache solutions here +Set from_vector(const Partition& partition, const SetAF& set_fact) { + Set partition_set = set_fact.createSet(); + for (size_t i = 0; i < partition.size(); i++) { + partition_set.emplace(partition[i]); + } + + return partition_set; +} + + +Partition to_vector(const Set& partition_set) +{ + Partition partition; + for (auto set_piece : partition_set) { + partition.push_back(move(set_piece)); + } + + return partition; +} + + +PartitionMap best_initial_partition(WeightedSBGraph& graph, unsigned number_of_partitions, + const InitialPartitionStrategy strategy) { - std::vector partition_maps = make_initial_partitions(graph, number_of_partitions); + logging::sbg_log << "computing strategy number " << strategy << endl; + std::vector partition_maps = make_initial_partitions(graph, number_of_partitions, strategy); auto& best_initial_partitions = partition_maps.front(); CommunicationCost comm_cost = CommunicationCost(graph, best_initial_partitions); - if (using_many_initial_partitions) { + if (strategy == InitialPartitionStrategy::ALL) { size_t best_communication_set_cardinality = get_partition_communication(graph, best_initial_partitions); for (unsigned i = 0; i < number_of_partitions; i++) { best_communication_set_cardinality += get_set_size(comm_cost.get_ec_by_partition_id(i)); diff --git a/algorithms/partitioner/partition_graph.hpp b/algorithms/partitioner/partition_graph.hpp index c06b9323..eed2d5b0 100644 --- a/algorithms/partitioner/partition_graph.hpp +++ b/algorithms/partitioner/partition_graph.hpp @@ -34,6 +34,15 @@ namespace sbg_partitioner { constexpr bool sanity_check_enabled = false; +enum InitialPartitionStrategy { + ALL = 0, + DFS_DISTRIBUTIVE_PREORDER = 1, + DFS_DISTRIBUTIVE_POSTORDER = 2, + DFS_GREEDY_PREORDER = 3, + DFS_GREEDY_POSTORDER =4 +}; + + /// @brief Converts a Partition element into a Set. /// @param partition - A list of SetPiece elements. /// @param set_fact - Factory to create sets. @@ -47,18 +56,11 @@ SBG::LIB::Set from_vector(const Partition& partition, const SBG::LIB::SetAF& set Partition to_vector(const SBG::LIB::Set& partition); -// I wish this was a separate function, not part of PartitionGraph but there were a lot of -// compile problems if partitions map object is created locally and OrdSet objects are added. -std::vector -make_initial_partitions( - SBG::LIB::WeightedSBGraph& graph, - unsigned number_of_partitions); - - PartitionMap best_initial_partition( SBG::LIB::WeightedSBGraph& graph, - unsigned number_of_partitions); + unsigned number_of_partitions, + const InitialPartitionStrategy strategy); /// Returns the connectivity set of a set of edges contained in map1 and map2 of diff --git a/algorithms/partitioner/partition_strategy.hpp b/algorithms/partitioner/partition_strategy.hpp index 29aebe52..d6f1d424 100644 --- a/algorithms/partitioner/partition_strategy.hpp +++ b/algorithms/partitioner/partition_strategy.hpp @@ -36,7 +36,7 @@ class PartitionStrategy public: PartitionStrategy() = default; - ~PartitionStrategy() = default; + virtual ~PartitionStrategy() = default; virtual void operator() (const SBG::LIB::SetPiece& node) = 0; diff --git a/test/partitioner/partitioner_test.cpp b/test/partitioner/partitioner_test.cpp index 4ced957d..df99540b 100644 --- a/test/partitioner/partitioner_test.cpp +++ b/test/partitioner/partitioner_test.cpp @@ -121,7 +121,7 @@ TEST(initial_partition, PartitionerTests) auto sb_graph = sbg_partitioner::build_sb_graph(get_full_file_name("air_conditioners_1000.json"), pw_fact); - sbg_partitioner::PartitionMap partition = sbg_partitioner::best_initial_partition(sb_graph, 4); + sbg_partitioner::PartitionMap partition = sbg_partitioner::best_initial_partition(sb_graph, 4, sbg_partitioner::InitialPartitionStrategy::ALL); auto expected_distributed_pre_order_0 = set_fact.createSet(); expected_distributed_pre_order_0.emplaceBack(Interval(0, 1, 249)); @@ -163,7 +163,7 @@ static void test_partitioning(const std::string& filename, int number_of_partiti UnordPWMapAF pw_fact(map_fact); auto sb_graph = sbg_partitioner::build_sb_graph(filename, pw_fact); - auto partitions = sbg_partitioner::best_initial_partition(sb_graph, number_of_partitions); + auto partitions = sbg_partitioner::best_initial_partition(sb_graph, number_of_partitions, sbg_partitioner::InitialPartitionStrategy::ALL); sbg_partitioner::kl_sbg_imbalance_partitioner(sb_graph, partitions, 0.0); sbg_partitioner::sanity_check(sb_graph, partitions, number_of_partitions); From cb4d699de91b49d8501ff40b020b2ece55fb75c8 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 25 Aug 2025 14:01:53 -0300 Subject: [PATCH 05/13] add multithreading flag --- .../partitioner/kernighan_lin_partitioner.cpp | 6 ++---- .../partitioner/kernighan_lin_partitioner.hpp | 3 ++- algorithms/partitioner/main.cpp | 15 +++++++++++---- test/partitioner/partitioner_test.cpp | 3 ++- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/algorithms/partitioner/kernighan_lin_partitioner.cpp b/algorithms/partitioner/kernighan_lin_partitioner.cpp index 1929399a..f345af9d 100644 --- a/algorithms/partitioner/kernighan_lin_partitioner.cpp +++ b/algorithms/partitioner/kernighan_lin_partitioner.cpp @@ -44,8 +44,6 @@ using ec_ic = std::pair; // Using unnamed namespace to define functions with internal linkage namespace { -constexpr bool multithreading_enabled = false; - pair compute_lmin_lmax(const WeightedSBGraph& graph, unsigned number_of_partitions, const float imbalance_epsilon, const SetAF& set_fact) { @@ -582,7 +580,7 @@ string get_pretty_sb_graph(const SBG::LIB::SBG& g) return json_data; } -void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& partitions, const float imbalance_epsilon) +void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& partitions, const float imbalance_epsilon, const bool enable_multithreading) { auto [LMin, LMax] = imbalance_epsilon > 0.0 ? compute_lmin_lmax(graph, partitions.size(), imbalance_epsilon, graph.fact()) : make_pair(0, 0); @@ -596,7 +594,7 @@ void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& pa change = false; KLSbgPartitionerResult best_gain; - if (multithreading_enabled) { + if (enable_multithreading) { best_gain = kl_sbg_partitioner_multithreading(graph, partitions, cost_matrix, LMin, LMax, gains); } else { best_gain = kl_sbg_partitioner_function(graph, partitions, cost_matrix, LMin, LMax, gains); diff --git a/algorithms/partitioner/kernighan_lin_partitioner.hpp b/algorithms/partitioner/kernighan_lin_partitioner.hpp index 66fa5e03..925d8bc8 100644 --- a/algorithms/partitioner/kernighan_lin_partitioner.hpp +++ b/algorithms/partitioner/kernighan_lin_partitioner.hpp @@ -37,7 +37,8 @@ namespace sbg_partitioner { void kl_sbg_imbalance_partitioner( const SBG::LIB::WeightedSBGraph& graph, PartitionMap& partitions, - const float imbalance_epsilon); + const float imbalance_epsilon, + const bool enable_multithreading); /** diff --git a/algorithms/partitioner/main.cpp b/algorithms/partitioner/main.cpp index 2e7e626b..b8677a3e 100644 --- a/algorithms/partitioner/main.cpp +++ b/algorithms/partitioner/main.cpp @@ -52,6 +52,8 @@ static void usage() cout << "-d, --directory Directory with partitions obtianed by other partitioners, " "we want to run quality metrics against them." << endl; + cout << "-t, --enable-multithreading Enable multithreading during optimization. WARNING: multithreading " + "is in experimental stage." << endl; cout << "-i, --initial-partition-strategy Choose a particular initial partition strategy. If this " "flag is disbaled, all strategies will be computed and the best partition will be chosen.\n" "\tValue\tSearching algorithm\tStrategy\tOrder\n" @@ -101,17 +103,18 @@ int main(int argc, char** argv) optional output_sb_graph = nullopt; optional epsilon = nullopt; InitialPartitionStrategy initial_partition_strategy = InitialPartitionStrategy::ALL; + bool enable_multithreading = false; bool compute_metrics = false; while (true) { static struct option long_options[] = {{"filename", required_argument, 0, 'f'}, {"partitions", required_argument, 0, 'p'}, {"output-file", required_argument, 0, 'g'}, {"output-graph", required_argument, 0, 'o'}, {"compute-metrics", no_argument, 0, 'm'}, {"directory", required_argument, 0, 'd'}, - {"initial-partition-strategy", required_argument, 0, 'i'}, - {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}}; + {"initial-partition-strategy", required_argument, 0, 'i'}, {"enable-multithreading", no_argument, 0, 't'}, + {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}}; int option_index = 0; - opt = getopt_long(argc, argv, "f:p:e:o:g:d:i:mvh:", long_options, &option_index); + opt = getopt_long(argc, argv, "f:p:e:o:g:d:i:tmvh:", long_options, &option_index); if (opt == EOF) break; switch (opt) { @@ -161,6 +164,10 @@ int main(int argc, char** argv) } break; + case 't': + enable_multithreading = true; + break; + case 'v': version(); exit(0); @@ -215,7 +222,7 @@ int main(int argc, char** argv) auto start_partitionate = chrono::high_resolution_clock::now(); auto partitions = best_initial_partition(sb_graph, *number_of_partitions, initial_partition_strategy); - kl_sbg_imbalance_partitioner(sb_graph, partitions, *epsilon); + kl_sbg_imbalance_partitioner(sb_graph, partitions, *epsilon, enable_multithreading); auto end_partitionate = chrono::high_resolution_clock::now(); auto time_to_partitionate = chrono::duration(end_partitionate - start_partitionate).count(); diff --git a/test/partitioner/partitioner_test.cpp b/test/partitioner/partitioner_test.cpp index df99540b..b10a61fa 100644 --- a/test/partitioner/partitioner_test.cpp +++ b/test/partitioner/partitioner_test.cpp @@ -164,7 +164,8 @@ static void test_partitioning(const std::string& filename, int number_of_partiti auto sb_graph = sbg_partitioner::build_sb_graph(filename, pw_fact); auto partitions = sbg_partitioner::best_initial_partition(sb_graph, number_of_partitions, sbg_partitioner::InitialPartitionStrategy::ALL); - sbg_partitioner::kl_sbg_imbalance_partitioner(sb_graph, partitions, 0.0); + constexpr bool enable_multithreading = false; + sbg_partitioner::kl_sbg_imbalance_partitioner(sb_graph, partitions, 0.0, enable_multithreading); sbg_partitioner::sanity_check(sb_graph, partitions, number_of_partitions); } From 6b407f57ca591cd5c77b0f2c57275a741c6486d5 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 25 Aug 2025 14:03:15 -0300 Subject: [PATCH 06/13] make cost by set_piece static --- algorithms/partitioner/communication_cost.cpp | 2 ++ algorithms/partitioner/communication_cost.hpp | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/algorithms/partitioner/communication_cost.cpp b/algorithms/partitioner/communication_cost.cpp index 55365b49..0dd7acea 100644 --- a/algorithms/partitioner/communication_cost.cpp +++ b/algorithms/partitioner/communication_cost.cpp @@ -28,6 +28,8 @@ using namespace SBG::LIB; namespace sbg_partitioner { +unordered_map CommunicationCost::_communication_by_set_piece = {}; + namespace internal { diff --git a/algorithms/partitioner/communication_cost.hpp b/algorithms/partitioner/communication_cost.hpp index 9e727f78..d6898bfe 100644 --- a/algorithms/partitioner/communication_cost.hpp +++ b/algorithms/partitioner/communication_cost.hpp @@ -80,7 +80,8 @@ class CommunicationCost { const SBG::LIB::WeightedSBGraph& _graph; // read-only members PartitionMap _partitions; - std::unordered_map _communication_by_set_piece; + // since communication is independent from the partitions, we can share it between many objects + static std::unordered_map _communication_by_set_piece; std::vector> _cost_by_partition; std::vector> _ec_cost_by_interval; std::vector> _ic_cost_by_interval; From f2c9382c028408794d2f78d8b5080635125625bc Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 25 Aug 2025 14:42:27 -0300 Subject: [PATCH 07/13] compute best initial partition properly --- algorithms/partitioner/partition_graph.cpp | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/algorithms/partitioner/partition_graph.cpp b/algorithms/partitioner/partition_graph.cpp index 01b708da..e46383df 100644 --- a/algorithms/partitioner/partition_graph.cpp +++ b/algorithms/partitioner/partition_graph.cpp @@ -161,26 +161,36 @@ PartitionMap best_initial_partition(WeightedSBGraph& graph, unsigned number_of_p std::vector partition_maps = make_initial_partitions(graph, number_of_partitions, strategy); auto& best_initial_partitions = partition_maps.front(); - CommunicationCost comm_cost = CommunicationCost(graph, best_initial_partitions); + unique_ptr comm_cost = make_unique(graph, best_initial_partitions); if (strategy == InitialPartitionStrategy::ALL) { - size_t best_communication_set_cardinality = get_partition_communication(graph, best_initial_partitions); + + auto best_communication_set = graph.fact().createSet(); for (unsigned i = 0; i < number_of_partitions; i++) { - best_communication_set_cardinality += get_set_size(comm_cost.get_ec_by_partition_id(i)); + best_communication_set = best_communication_set.cup(comm_cost->get_ec_by_partition_id(i)); } + size_t best_communication_set_size = best_communication_set.cardinal(); for (size_t i = 1; i < partition_maps.size(); i++) { auto temp_intial_partitions = partition_maps[i]; - CommunicationCost temp_comm_cost = CommunicationCost(graph, temp_intial_partitions); - size_t temp_partition_comm_size = get_partition_communication(graph, temp_intial_partitions); + unique_ptr temp_comm_cost = make_unique(graph, temp_intial_partitions); + + auto temp_partition_comm = graph.fact().createSet(); for (unsigned i = 0; i < number_of_partitions; i++) { - temp_partition_comm_size += get_set_size(comm_cost.get_ec_by_partition_id(i)); + temp_partition_comm = temp_partition_comm.cup(comm_cost->get_ec_by_partition_id(i)); + } + + size_t temp_intial_partitions_size = temp_partition_comm.cardinal(); + + if (temp_intial_partitions_size < best_communication_set_size) { + comm_cost = move(temp_comm_cost); + best_initial_partitions = move(partition_maps[i]); } } - logging::sbg_log << "Best is " << best_initial_partitions << " with communication " << best_communication_set_cardinality << endl; + logging::sbg_log << "Best is " << best_initial_partitions << " with communication " << best_communication_set << endl; } - set_communication_cost(comm_cost); + set_communication_cost(move(comm_cost)); return best_initial_partitions; } From 473c63a588a25e053b06293db2951ab2744a22cc Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Mon, 25 Aug 2025 16:50:46 -0300 Subject: [PATCH 08/13] deal with multithreading propely --- algorithms/partitioner/communication_cost.cpp | 66 +++++++++++++++--- algorithms/partitioner/communication_cost.hpp | 68 ++++++++++++++++--- .../partitioner/kernighan_lin_partitioner.cpp | 12 ++-- algorithms/partitioner/main.cpp | 2 +- algorithms/partitioner/partition_graph.cpp | 6 +- algorithms/partitioner/partition_graph.hpp | 3 +- test/partitioner/partitioner_test.cpp | 5 +- 7 files changed, 133 insertions(+), 29 deletions(-) diff --git a/algorithms/partitioner/communication_cost.cpp b/algorithms/partitioner/communication_cost.cpp index 0dd7acea..a882a44b 100644 --- a/algorithms/partitioner/communication_cost.cpp +++ b/algorithms/partitioner/communication_cost.cpp @@ -33,7 +33,8 @@ unordered_map CommunicationCost::_communication_by_ namespace internal { -static CommunicationCost* cost_matrix = nullptr; +// the only real instance +CommunicationCostPtr cost_matrix = nullptr; namespace { @@ -59,8 +60,9 @@ Set set_piece_communication(const SetPiece& nodes, const WeightedSBGraph& graph) CommunicationCost::CommunicationCost(const WeightedSBGraph& graph, PartitionMap partitions) - : _graph(graph), - _partitions(partitions) + : ICommunicationCost(), + _graph(graph), + _partitions(partitions) { initialize(); } @@ -77,8 +79,11 @@ void CommunicationCost::initialize() Set internal_communication_partition_i = _graph.fact().createSet(); for (const auto& node : _partitions.at(i)) { - auto node_edges = internal::set_piece_communication(node, _graph); - _communication_by_set_piece.insert({node, node_edges}); + if (_communication_by_set_piece.find(node) == _communication_by_set_piece.end()) { + _communication_by_set_piece.insert({node, internal::set_piece_communication(node, _graph)}); + } + + const auto& node_edges = _communication_by_set_piece.at(node); internal_communication_partition_i = node_edges.intersection(partition_i_communication).cup(internal_communication_partition_i); partition_i_communication = partition_i_communication.cup(node_edges); } @@ -182,12 +187,57 @@ Set CommunicationCost::get_ic_by_interval(unsigned partition_id, const SetPiece& } -void set_communication_cost(CommunicationCost& cost_matrix) + +CommunicationCostSync::CommunicationCostSync(const WeightedSBGraph& graph, PartitionMap partitions) + :ICommunicationCost(), + _comm_cost(graph, partitions) +{} + + +void CommunicationCostSync::update_partitions(PartitionMap& partitions, optional>> modified_partitions) +{ + const lock_guard lock(_mutex); + _comm_cost.update_partitions(partitions, modified_partitions); +} + + +Set CommunicationCostSync::get_ec_by_partition_id(unsigned partition_id) +{ + const lock_guard lock(_mutex); + return _comm_cost.get_ec_by_partition_id(partition_id); +} + + +Set CommunicationCostSync::get_ec_by_interval(unsigned partition_id, const SetPiece& nodes) +{ + const lock_guard lock(_mutex); + return _comm_cost.get_ec_by_interval(partition_id, nodes); +} + + +Set CommunicationCostSync::get_ic_by_interval(unsigned partition_id, const SetPiece& nodes) +{ + const lock_guard lock(_mutex); + return _comm_cost.get_ic_by_interval(partition_id, nodes); +} + + + +CommunicationCostPtr create_communication_cost(const WeightedSBGraph& graph, PartitionMap partitions, bool multithreading_enabled) +{ + if (multithreading_enabled) { + return make_unique(graph, partitions); + } else { + return make_unique(graph, partitions); + } +} + +void set_communication_cost(CommunicationCostPtr&& cost_matrix) { - internal::cost_matrix = new CommunicationCost(cost_matrix); + internal::cost_matrix = move(cost_matrix); } -CommunicationCost& get_communication_cost() +ICommunicationCost& get_communication_cost() { assert(internal::cost_matrix); return *internal::cost_matrix; diff --git a/algorithms/partitioner/communication_cost.hpp b/algorithms/partitioner/communication_cost.hpp index d6898bfe..6bb00df0 100644 --- a/algorithms/partitioner/communication_cost.hpp +++ b/algorithms/partitioner/communication_cost.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -30,9 +31,12 @@ namespace sbg_partitioner { -class CommunicationCost { + +class ICommunicationCost { public: - CommunicationCost(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); + ICommunicationCost() = default; + + virtual ~ICommunicationCost() = default; /** * @brief After partitions are modified, external and internal cost must be updated and this function takes care of it. @@ -41,7 +45,7 @@ class CommunicationCost { * @param partitions - new partition of the graph nodes. * @param modified_partitions - [optional] partitions that were updated. */ - void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); + virtual void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt) = 0; /** * It returns the edges that communicate nodes in partition `partition_id` with others. @@ -51,7 +55,7 @@ class CommunicationCost { * @return External edges of the given partition. * @note These values are pre-computed when the object is created or partitions are updated. */ - SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id); // non-const since _cost_by_partition may be updated + virtual SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id) = 0; // non-const since _cost_by_partition may be updated /** * It returns the edges that communicate the set piece nodes in partition `partition_id` with other partitions. @@ -63,7 +67,7 @@ class CommunicationCost { * @note the member function is non-const since _ec_cost_by_interval and _ic_cost_by_interval may be updated to prevent to be recomputed. */ - SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + virtual SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) = 0; /** * It returns the edges that communicate the set piece nodes with other nodes in partition `partition_id`. @@ -74,7 +78,29 @@ class CommunicationCost { * @return Internal edges of the given set piece. * @note the member function is non-const since _ec_cost_by_interval and _ic_cost_by_interval may be updated to prevent to be recomputed. */ - SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + virtual SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) = 0; +}; + + +/// Communication cost pointer +typedef std::unique_ptr CommunicationCostPtr; + +/** + * The actual communication cost class. + */ +class CommunicationCost : public ICommunicationCost { +public: + CommunicationCost(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); + + virtual ~CommunicationCost() = default; + + virtual void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); + + virtual SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id); // non-const since _cost_by_partition may be updated + + virtual SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + + virtual SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); private: const SBG::LIB::WeightedSBGraph& _graph; // read-only members @@ -90,14 +116,40 @@ class CommunicationCost { std::pair compute_ec_ic(unsigned partition_id, const SBG::LIB::SetPiece& nodes); }; + +/** + * Communication cost to run optimization using multithreading. + */ +class CommunicationCostSync : public ICommunicationCost { +public: + CommunicationCostSync(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); + + virtual ~CommunicationCostSync() = default; + + virtual void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); + + virtual SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id); // non-const since _cost_by_partition may be updated + + virtual SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + + virtual SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + +private: + CommunicationCost _comm_cost; + std::mutex _mutex; +}; + + +CommunicationCostPtr create_communication_cost(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions, bool multithreading_enabled); + /** * @brief Once communication cost object is created, it can be saved by calling this function to be used in the future. */ -void set_communication_cost(CommunicationCost& cost_matrix); +void set_communication_cost(CommunicationCostPtr&& cost_matrix); /** * @brief Global communication cost object, saved to prevent recomputing. */ -CommunicationCost& get_communication_cost(); +ICommunicationCost& get_communication_cost(); } \ No newline at end of file diff --git a/algorithms/partitioner/kernighan_lin_partitioner.cpp b/algorithms/partitioner/kernighan_lin_partitioner.cpp index f345af9d..afa0cc82 100644 --- a/algorithms/partitioner/kernighan_lin_partitioner.cpp +++ b/algorithms/partitioner/kernighan_lin_partitioner.cpp @@ -56,7 +56,7 @@ pair compute_lmin_lmax(const WeightedSBGraph& graph, unsigne return make_pair(LMin, LMax); } -pair generate_gain_matrix(const WeightedSBGraph& graph, CommunicationCost& cost_matrix, unsigned partition_a_id, +pair generate_gain_matrix(const WeightedSBGraph& graph, ICommunicationCost& cost_matrix, unsigned partition_a_id, Partition& partition_a, unsigned partition_b_id, Partition& partition_b, unsigned LMin, unsigned LMax) { @@ -295,7 +295,7 @@ void update_sum(int& par_sum, int g, int& max_par_sum, pair& max_par_s } -int kl_sbg_imbalance(const WeightedSBGraph& graph, CommunicationCost& cost_matrix, unsigned partition_a_id, Partition& partition_a, +int kl_sbg_imbalance(const WeightedSBGraph& graph, ICommunicationCost& cost_matrix, unsigned partition_a_id, Partition& partition_a, unsigned partition_b_id, Partition& partition_b, unsigned LMin, unsigned LMax) { #if PARTITION_IMBALANCE_DEBUG @@ -355,7 +355,7 @@ int kl_sbg_imbalance(const WeightedSBGraph& graph, CommunicationCost& cost_matri return max_par_sum; } -KLBipartResult kl_sbg_bipart_imbalance(const WeightedSBGraph& graph, CommunicationCost& cost_matrix, unsigned partition_a_id, +KLBipartResult kl_sbg_bipart_imbalance(const WeightedSBGraph& graph, ICommunicationCost& cost_matrix, unsigned partition_a_id, Partition& partition_a, unsigned partition_b_id, Partition& partition_b, unsigned LMin, unsigned LMax) { @@ -368,7 +368,7 @@ KLBipartResult kl_sbg_bipart_imbalance(const WeightedSBGraph& graph, Communicati return KLBipartResult{partition_a, partition_b, gain}; } -KLSbgPartitionerResult kl_sbg_partitioner_function(const WeightedSBGraph& graph, PartitionMap& partitions, CommunicationCost& cost_matrix, +KLSbgPartitionerResult kl_sbg_partitioner_function(const WeightedSBGraph& graph, PartitionMap& partitions, ICommunicationCost& cost_matrix, unsigned LMin, unsigned LMax, list& gains) { KLSbgPartitionerResult best_gain = KLSbgPartitionerResult{0, 0, -1, {}, {}}; @@ -408,7 +408,7 @@ KLSbgPartitionerResult kl_sbg_partitioner_function(const WeightedSBGraph& graph, } KLSbgPartitionerResult kl_sbg_partitioner_multithreading(const WeightedSBGraph& graph, PartitionMap& partitions, - CommunicationCost& cost_matrix, unsigned LMin, unsigned LMax, + ICommunicationCost& cost_matrix, unsigned LMin, unsigned LMax, list& gains) { list> workers; @@ -587,7 +587,7 @@ void kl_sbg_imbalance_partitioner(const WeightedSBGraph& graph, PartitionMap& pa bool change = true; int counter = 0; - CommunicationCost& cost_matrix = get_communication_cost(); + ICommunicationCost& cost_matrix = get_communication_cost(); list gains; while (change) { cout << "*****ITERATION NUMBER " << counter++ << endl; diff --git a/algorithms/partitioner/main.cpp b/algorithms/partitioner/main.cpp index b8677a3e..92efc05e 100644 --- a/algorithms/partitioner/main.cpp +++ b/algorithms/partitioner/main.cpp @@ -221,7 +221,7 @@ int main(int argc, char** argv) cout << "sb_graph: " << sb_graph << endl; auto start_partitionate = chrono::high_resolution_clock::now(); - auto partitions = best_initial_partition(sb_graph, *number_of_partitions, initial_partition_strategy); + auto partitions = best_initial_partition(sb_graph, *number_of_partitions, initial_partition_strategy, enable_multithreading); kl_sbg_imbalance_partitioner(sb_graph, partitions, *epsilon, enable_multithreading); auto end_partitionate = chrono::high_resolution_clock::now(); auto time_to_partitionate = chrono::duration(end_partitionate - start_partitionate).count(); diff --git a/algorithms/partitioner/partition_graph.cpp b/algorithms/partitioner/partition_graph.cpp index e46383df..9f1677ca 100644 --- a/algorithms/partitioner/partition_graph.cpp +++ b/algorithms/partitioner/partition_graph.cpp @@ -155,13 +155,13 @@ Partition to_vector(const Set& partition_set) PartitionMap best_initial_partition(WeightedSBGraph& graph, unsigned number_of_partitions, - const InitialPartitionStrategy strategy) + const InitialPartitionStrategy strategy, bool multithreading_enabled) { logging::sbg_log << "computing strategy number " << strategy << endl; std::vector partition_maps = make_initial_partitions(graph, number_of_partitions, strategy); auto& best_initial_partitions = partition_maps.front(); - unique_ptr comm_cost = make_unique(graph, best_initial_partitions); + CommunicationCostPtr comm_cost = create_communication_cost(graph, best_initial_partitions, multithreading_enabled); if (strategy == InitialPartitionStrategy::ALL) { auto best_communication_set = graph.fact().createSet(); @@ -172,7 +172,7 @@ PartitionMap best_initial_partition(WeightedSBGraph& graph, unsigned number_of_p for (size_t i = 1; i < partition_maps.size(); i++) { auto temp_intial_partitions = partition_maps[i]; - unique_ptr temp_comm_cost = make_unique(graph, temp_intial_partitions); + CommunicationCostPtr temp_comm_cost = create_communication_cost(graph, temp_intial_partitions, multithreading_enabled); auto temp_partition_comm = graph.fact().createSet(); for (unsigned i = 0; i < number_of_partitions; i++) { diff --git a/algorithms/partitioner/partition_graph.hpp b/algorithms/partitioner/partition_graph.hpp index eed2d5b0..d5c683f9 100644 --- a/algorithms/partitioner/partition_graph.hpp +++ b/algorithms/partitioner/partition_graph.hpp @@ -60,7 +60,8 @@ PartitionMap best_initial_partition( SBG::LIB::WeightedSBGraph& graph, unsigned number_of_partitions, - const InitialPartitionStrategy strategy); + const InitialPartitionStrategy strategy, + bool multithreading_enabled); /// Returns the connectivity set of a set of edges contained in map1 and map2 of diff --git a/test/partitioner/partitioner_test.cpp b/test/partitioner/partitioner_test.cpp index b10a61fa..03a87ebc 100644 --- a/test/partitioner/partitioner_test.cpp +++ b/test/partitioner/partitioner_test.cpp @@ -121,7 +121,8 @@ TEST(initial_partition, PartitionerTests) auto sb_graph = sbg_partitioner::build_sb_graph(get_full_file_name("air_conditioners_1000.json"), pw_fact); - sbg_partitioner::PartitionMap partition = sbg_partitioner::best_initial_partition(sb_graph, 4, sbg_partitioner::InitialPartitionStrategy::ALL); + constexpr bool enable_multithreading = false; + sbg_partitioner::PartitionMap partition = sbg_partitioner::best_initial_partition(sb_graph, 4, sbg_partitioner::InitialPartitionStrategy::ALL, enable_multithreading); auto expected_distributed_pre_order_0 = set_fact.createSet(); expected_distributed_pre_order_0.emplaceBack(Interval(0, 1, 249)); @@ -163,8 +164,8 @@ static void test_partitioning(const std::string& filename, int number_of_partiti UnordPWMapAF pw_fact(map_fact); auto sb_graph = sbg_partitioner::build_sb_graph(filename, pw_fact); - auto partitions = sbg_partitioner::best_initial_partition(sb_graph, number_of_partitions, sbg_partitioner::InitialPartitionStrategy::ALL); constexpr bool enable_multithreading = false; + auto partitions = sbg_partitioner::best_initial_partition(sb_graph, number_of_partitions, sbg_partitioner::InitialPartitionStrategy::ALL, enable_multithreading); sbg_partitioner::kl_sbg_imbalance_partitioner(sb_graph, partitions, 0.0, enable_multithreading); sbg_partitioner::sanity_check(sb_graph, partitions, number_of_partitions); From 57ee0b2c986dfb05a304d25927c357ade7b56903 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Tue, 26 Aug 2025 08:46:14 -0300 Subject: [PATCH 09/13] added typedef --- algorithms/partitioner/dfs_on_sbg.cpp | 4 ++-- algorithms/partitioner/dfs_on_sbg.hpp | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/algorithms/partitioner/dfs_on_sbg.cpp b/algorithms/partitioner/dfs_on_sbg.cpp index 50c04606..46f879bd 100644 --- a/algorithms/partitioner/dfs_on_sbg.cpp +++ b/algorithms/partitioner/dfs_on_sbg.cpp @@ -41,7 +41,7 @@ void initialize_partitioning(SBG::LIB::WeightedSBGraph& graph, unsigned number_o sort_object = DFS(graph, number_of_partitions); } -void add_strategy(unique_ptr&& strategy, bool pre_order) { sort_object.add_partition_strategy(move(strategy), pre_order); } +void add_strategy(PartitionStrategyPtr&& strategy, bool pre_order) { sort_object.add_partition_strategy(move(strategy), pre_order); } vector>> partitionate() { @@ -167,7 +167,7 @@ void DFS::iterate() } } -void DFS::add_partition_strategy(unique_ptr&& strategy, bool pre_order) +void DFS::add_partition_strategy(PartitionStrategyPtr&& strategy, bool pre_order) { if (pre_order) { _partition_strategy_pre_order.push_back(move(strategy)); diff --git a/algorithms/partitioner/dfs_on_sbg.hpp b/algorithms/partitioner/dfs_on_sbg.hpp index 28e008f3..522df904 100644 --- a/algorithms/partitioner/dfs_on_sbg.hpp +++ b/algorithms/partitioner/dfs_on_sbg.hpp @@ -33,11 +33,13 @@ namespace sbg_partitioner { +typedef std::unique_ptr PartitionStrategyPtr; + namespace search { void initialize_partitioning(SBG::LIB::WeightedSBGraph& graph, unsigned number_of_partitions); -void add_strategy(std::unique_ptr&& strategy, bool pre_order); +void add_strategy(PartitionStrategyPtr&& strategy, bool pre_order); std::vector>> partitionate(); @@ -63,7 +65,7 @@ class DFS { std::vector>> partitions() const; /// @note strategy object should live while this class does - void add_partition_strategy(std::unique_ptr&& strategy, bool pre_order); + void add_partition_strategy(PartitionStrategyPtr&& strategy, bool pre_order); private: typedef SBG::LIB::Set::Iterator node_identifier; @@ -81,8 +83,8 @@ class DFS { SBG::LIB::WeightedSBGraph* _graph; SBG::LIB::Set _nodes; - std::vector> _partition_strategy_pre_order; - std::vector> _partition_strategy_post_order; + std::vector _partition_strategy_pre_order; + std::vector _partition_strategy_post_order; void initialize_adjacents(); From 5875dbe0cd7d16fdca393195f472bd00e46bb2bc Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Tue, 26 Aug 2025 09:40:27 -0300 Subject: [PATCH 10/13] remove warning message --- algorithms/partitioner/main.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/algorithms/partitioner/main.cpp b/algorithms/partitioner/main.cpp index 92efc05e..f3504042 100644 --- a/algorithms/partitioner/main.cpp +++ b/algorithms/partitioner/main.cpp @@ -52,10 +52,9 @@ static void usage() cout << "-d, --directory Directory with partitions obtianed by other partitioners, " "we want to run quality metrics against them." << endl; - cout << "-t, --enable-multithreading Enable multithreading during optimization. WARNING: multithreading " - "is in experimental stage." << endl; + cout << "-t, --enable-multithreading Enable multithreading during optimization." << endl; cout << "-i, --initial-partition-strategy Choose a particular initial partition strategy. If this " - "flag is disbaled, all strategies will be computed and the best partition will be chosen.\n" + "flag is disabaled, all strategies will be computed and the best partition will be chosen.\n" "\tValue\tSearching algorithm\tStrategy\tOrder\n" "\t0\tDepth first search\tDistributive\tpreorder\n" "\t1\tDepth first search\tDistributive\tpostorder\n" From 8ddc9b05bc85bc48ad39946870870323b13ddb82 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Fri, 29 Aug 2025 14:44:28 -0300 Subject: [PATCH 11/13] removed virtual and added override in inherited classes --- algorithms/partitioner/communication_cost.hpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/algorithms/partitioner/communication_cost.hpp b/algorithms/partitioner/communication_cost.hpp index 6bb00df0..b27573bb 100644 --- a/algorithms/partitioner/communication_cost.hpp +++ b/algorithms/partitioner/communication_cost.hpp @@ -92,15 +92,15 @@ class CommunicationCost : public ICommunicationCost { public: CommunicationCost(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); - virtual ~CommunicationCost() = default; + ~CommunicationCost() = default; - virtual void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); + void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt) override; - virtual SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id); // non-const since _cost_by_partition may be updated + SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id) override; // non-const since _cost_by_partition may be updated - virtual SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) override; - virtual SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) override; private: const SBG::LIB::WeightedSBGraph& _graph; // read-only members @@ -124,15 +124,15 @@ class CommunicationCostSync : public ICommunicationCost { public: CommunicationCostSync(const SBG::LIB::WeightedSBGraph& graph, PartitionMap partitions); - virtual ~CommunicationCostSync() = default; + ~CommunicationCostSync() = default; - virtual void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt); + void update_partitions(PartitionMap& partitions, std::optional>> modified_partitions = std::nullopt) override; - virtual SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id); // non-const since _cost_by_partition may be updated + SBG::LIB::Set get_ec_by_partition_id(unsigned partition_id) override; // non-const since _cost_by_partition may be updated - virtual SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + SBG::LIB::Set get_ec_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) override; - virtual SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes); + SBG::LIB::Set get_ic_by_interval(unsigned partition_id, const SBG::LIB::SetPiece& nodes) override; private: CommunicationCost _comm_cost; From a63c36e49fbdcfb20cd2384d14e19d59b55846d1 Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Wed, 3 Sep 2025 17:04:48 -0300 Subject: [PATCH 12/13] fixed GainObjectImbalance initialization, gain was taken as 0 --- .../partitioner/kernighan_lin_partitioner.cpp | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/algorithms/partitioner/kernighan_lin_partitioner.cpp b/algorithms/partitioner/kernighan_lin_partitioner.cpp index afa0cc82..b7c2bad8 100644 --- a/algorithms/partitioner/kernighan_lin_partitioner.cpp +++ b/algorithms/partitioner/kernighan_lin_partitioner.cpp @@ -61,19 +61,8 @@ pair generate_gain_matrix(const Weight unsigned LMax) { SBG::Util::Internal::TimeProfiler profiler("generate_gain_matrix"); - const auto& fact = graph.fact(); // create the max_gain object with a dummy initialization, any gain will be greater than -infinity - GainObjectImbalance max_gain = GainObjectImbalance{ - numeric_limits::infinity(), - numeric_limits::infinity(), - -numeric_limits::infinity(), - fact.createSet(), - fact.createSet(), - 0, - fact.createSet(), - fact.createSet(), - 0 - }; + optional max_gain = nullopt; CostMatrixImbalance local_cost_matrix; for (size_t i = 0; i < partition_a.size(); i++) { @@ -104,13 +93,15 @@ pair generate_gain_matrix(const Weight int gain = ec_edges.cardinal() - ic_edges.cardinal(); local_cost_matrix.emplace_back(i, j, gain, ec_i_a, ic_i_a, set_i_a.cardinal(), ec_j_b, ic_j_b, set_j_b.cardinal()); - if (local_cost_matrix.back().gain > max_gain.gain) { + if ((not max_gain) or local_cost_matrix.back().gain > max_gain->gain) { max_gain = local_cost_matrix.back(); } } } - return { max_gain, local_cost_matrix }; + assert(max_gain); + + return { *max_gain, local_cost_matrix }; } // Partition a and b (A_c and B_c in the definition) are the remining nodes to be visited, not the actual partitions @@ -230,7 +221,7 @@ GainObjectImbalance update_diff(CostMatrixImbalance& cost_matrix, Partition& rem } // using a reference to copy the element only once when returning - GainObjectImbalance& max_gain_object = cost_matrix.front(); + optional max_gain_object = nullopt; for (auto g : cost_matrix) { bool change = false; @@ -271,17 +262,19 @@ GainObjectImbalance update_diff(CostMatrixImbalance& cost_matrix, Partition& rem new_cost_matrix.push_back(move(g)); - if (new_cost_matrix.back().gain > max_gain_object.gain) { + if ((not max_gain_object) or new_cost_matrix.back().gain > max_gain_object->gain) { max_gain_object = new_cost_matrix.back(); } } cost_matrix = new_cost_matrix; + assert(max_gain_object); + #if PARTITION_IMBALANCE_DEBUG logging::sbg_log << remaining_partition_a << ", " << remaining_partition_b << ", " << gain_object << ", " << cost_matrix << endl; #endif - return max_gain_object; + return *max_gain_object; } From 6a67c64a28c4bb17c3924ae0422fe0ab9cea7c3b Mon Sep 17 00:00:00 2001 From: Franco Sansone Date: Wed, 3 Sep 2025 17:51:06 -0300 Subject: [PATCH 13/13] avoid oversizing external communication --- .../partitioner/partition_metrics_api.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/algorithms/partitioner/partition_metrics_api.cpp b/algorithms/partitioner/partition_metrics_api.cpp index 6e6454c8..865bcb81 100644 --- a/algorithms/partitioner/partition_metrics_api.cpp +++ b/algorithms/partitioner/partition_metrics_api.cpp @@ -36,16 +36,17 @@ namespace metrics { namespace { Set get_edge_cut( - const Set& partition_a, - const Set& partition_b, + const Set& partition, const PWMap& maps_1, const PWMap& maps_2, SBG::LIB::SetAF& set_fact) { - auto d = maps_1.preImage(partition_a); + auto d = maps_1.preImage(partition); auto im = maps_2.image(d); - auto ec_nodes = im.intersection(partition_b); + auto ec_nodes = im.difference(partition); auto external_communication = maps_2.preImage(ec_nodes); + // avoid oversizing external communication + external_communication = external_communication.intersection(d); return external_communication; } @@ -132,12 +133,9 @@ int edge_cut(const PartitionMap& partitions, const WeightedSBGraph& sb_graph, Se const auto& maps_1 = sb_graph.map1(); const auto& maps_2 = sb_graph.map2(); for (size_t i = 0; i < partitions.size(); i++) { - Set partition_1 = from_vector(partitions.at(i), set_fact); - for (size_t j = i + 1; j < partitions.size(); j++) { - Set partition_2 = from_vector(partitions.at(j), set_fact); - ec = ec.cup(get_edge_cut(partition_1, partition_2, maps_1, maps_2, set_fact)); - ec = ec.cup(get_edge_cut(partition_1, partition_2, maps_2, maps_1, set_fact)); - } + Set partition = from_vector(partitions.at(i), set_fact); + ec = ec.cup(get_edge_cut(partition, maps_1, maps_2, set_fact)); + ec = ec.cup(get_edge_cut(partition, maps_2, maps_1, set_fact)); } int weight = get_edge_set_cost(ec, sb_graph.get_edge_costs());