#include <ParCommGraph.hpp>

Collaboration diagram for moab::ParCommGraph:

Public Types
enum	TypeGraph { INITIAL_MIGRATE , COVERAGE , DOF_BASED }

Public Member Functions
virtual	~ParCommGraph ()

	ParCommGraph (MPI_Comm joincomm, MPI_Group group1, MPI_Group group2, int coid1, int coid2)
	collective constructor, will be called on all sender tasks and receiver tasks More...

	ParCommGraph (const ParCommGraph &)
	copy constructor will copy only the senders, receivers, compid1, etc More...

ErrorCode	compute_trivial_partition (std::vector< int > &numElemsPerTaskInGroup1)
	Based on the number of elements on each task in group 1, partition for group 2, trivially. More...

ErrorCode	pack_receivers_graph (std::vector< int > &packed_recv_array)
	pack information about receivers view of the graph, for future sending to receiver root More...

bool	is_root_sender ()

bool	is_root_receiver ()

int	sender (int index)

int	receiver (int index)

int	get_component_id1 ()

int	get_component_id2 ()

int	get_context_id ()

void	set_context_id (int other_id)

EntityHandle	get_cover_set ()

void	set_cover_set (EntityHandle cover)

ErrorCode	split_owned_range (int sender_rank, Range &owned)

ErrorCode	split_owned_range (Range &owned)

ErrorCode	send_graph (MPI_Comm jcomm)

ErrorCode	send_graph_partition (ParallelComm *pco, MPI_Comm jcomm)

ErrorCode	send_mesh_parts (MPI_Comm jcomm, ParallelComm *pco, Range &owned)

ErrorCode	receive_comm_graph (MPI_Comm jcomm, ParallelComm *pco, std::vector< int > &pack_array)

ErrorCode	receive_mesh (MPI_Comm jcomm, ParallelComm *pco, EntityHandle local_set, std::vector< int > &senders_local)

ErrorCode	release_send_buffers ()

ErrorCode	send_tag_values (MPI_Comm jcomm, ParallelComm *pco, Range &owned, std::vector< Tag > &tag_handles)

ErrorCode	receive_tag_values (MPI_Comm jcomm, ParallelComm *pco, Range &owned, std::vector< Tag > &tag_handles)

const std::vector< int > &	senders ()

const std::vector< int > &	receivers ()

ErrorCode	settle_send_graph (TupleList &TLcovIDs)

void	SetReceivingAfterCoverage (std::map< int, std::set< int > > &idsFromProcs)

void	settle_comm_by_ids (int comp, TupleList &TLBackToComp, std::vector< int > &valuesComp)

ErrorCode	compute_partition (ParallelComm *pco, Range &owned, int met)

ErrorCode	dump_comm_information (std::string prefix, int is_send)

Private Member Functions
void	find_group_ranks (MPI_Group group, MPI_Comm join, std::vector< int > &ranks)
	find ranks of a group with respect to an encompassing communicator More...

Private Attributes
MPI_Comm	comm

std::vector< int >	senderTasks

std::vector< int >	receiverTasks

bool	rootSender

bool	rootReceiver

int	rankInGroup1

int	rankInGroup2

int	rankInJoin

int	joinSize

int	compid1

int	compid2

int	context_id

EntityHandle	cover_set

std::map< int, std::vector< int > >	recv_graph

std::map< int, std::vector< int > >	recv_sizes

std::map< int, std::vector< int > >	sender_graph

std::map< int, std::vector< int > >	sender_sizes

std::vector< ParallelComm::Buffer * >	localSendBuffs

int *	comm_graph

std::map< int, Range >	split_ranges

std::vector< MPI_Request >	sendReqs

std::vector< int >	corr_tasks

std::vector< int >	corr_sizes

TypeGraph	graph_type

std::map< int, std::vector< int > >	involved_IDs_map

std::map< int, std::vector< int > >	map_index

std::map< int, std::vector< int > >	map_ptr

Detailed Description

Definition at line 55 of file ParCommGraph.hpp.

Member Enumeration Documentation

◆ TypeGraph

enum moab::ParCommGraph::TypeGraph

Enumerator
INITIAL_MIGRATE
COVERAGE
DOF_BASED

Definition at line 58 of file ParCommGraph.hpp.

     {
         INITIAL_MIGRATE,
         COVERAGE,
         DOF_BASED
     };

Constructor & Destructor Documentation

◆ ~ParCommGraph()

moab::ParCommGraph::~ParCommGraph ( )

virtual

Definition at line 73 of file ParCommGraph.cpp.

 {
     // TODO Auto-generated destructor stub
 }

◆ ParCommGraph() [1/2]

moab::ParCommGraph::ParCommGraph	(	MPI_Comm	joincomm,
		MPI_Group	group1,
		MPI_Group	group2,
		int	coid1,
		int	coid2
	)

collective constructor, will be called on all sender tasks and receiver tasks

Parameters

[in]	joincomm	joint MPI communicator that covers both sender and receiver MPI groups
[in]	group1	MPI group formed with sender tasks; (sender usually loads the mesh in a migrate scenario)
[in]	group2	MPI group formed with receiver tasks; (receiver usually receives the mesh in a migrate scenario)
[in]	coid1	sender component unique identifier in a coupled application (in climate simulations could be the Atmosphere Comp id = 5 or Ocean Comp ID , 17)
[in]	coid2	receiver component unique identifier in a coupled application (it is usually the coupler, 2, in E3SM)

this graph will be formed on sender and receiver tasks, and, in principle, will hold info about how the local entities are distributed on the other side

Its major role is to help in migration of data, from component to the coupler and vice-versa; Each local entity has a corresponding task (or tasks) on the other side, to where the data needs to be sent

important data stored in ParCommGraph, immediately it is created

all sender and receiver tasks ids, with respect to the joint communicator
local rank in sender and receiver group (-1 if not part of the respective group)
rank in the joint communicator (from 0)

Definition at line 20 of file ParCommGraph.cpp.

     : comm( joincomm ), compid1( coid1 ), compid2( coid2 )
 {
     // find out the tasks from each group, in the joint communicator
     find_group_ranks( group1, comm, senderTasks );
     find_group_ranks( group2, comm, receiverTasks );
  
     rootSender = rootReceiver = false;
     rankInGroup1 = rankInGroup2 = rankInJoin = -1;  // not initialized, or not part of the group
  
     int mpierr = MPI_Group_rank( group1, &rankInGroup1 );
     if( MPI_SUCCESS != mpierr || rankInGroup1 == MPI_UNDEFINED ) rankInGroup1 = -1;
  
     mpierr = MPI_Group_rank( group2, &rankInGroup2 );
     if( MPI_SUCCESS != mpierr || rankInGroup2 == MPI_UNDEFINED ) rankInGroup2 = -1;
  
     mpierr = MPI_Comm_rank( comm, &rankInJoin );
     if( MPI_SUCCESS != mpierr )  // it should be a fatal error
         rankInJoin = -1;
  
     mpierr = MPI_Comm_size( comm, &joinSize );
     if( MPI_SUCCESS != mpierr )  // it should be a fatal error
         joinSize = -1;
  
     if( 0 == rankInGroup1 ) rootSender = true;
     if( 0 == rankInGroup2 ) rootReceiver = true;
     graph_type = INITIAL_MIGRATE;  // 0
     comm_graph = NULL;
     context_id = -1;
     cover_set  = 0;  // refers to nothing yet
 }

References comm, comm_graph, context_id, cover_set, find_group_ranks(), graph_type, INITIAL_MIGRATE, joinSize, rankInGroup1, rankInGroup2, rankInJoin, receiverTasks, rootReceiver, rootSender, and senderTasks.

◆ ParCommGraph() [2/2]

moab::ParCommGraph::ParCommGraph ( const ParCommGraph & src )

copy constructor will copy only the senders, receivers, compid1, etc

Definition at line 53 of file ParCommGraph.cpp.

 {
     comm          = src.comm;
     senderTasks   = src.senderTasks;    // these are the sender tasks in joint comm
     receiverTasks = src.receiverTasks;  // these are all the receiver tasks in joint comm
     rootSender    = src.rootSender;
     rootReceiver  = src.rootReceiver;
     rankInGroup1  = src.rankInGroup1;
     rankInGroup2  = src.rankInGroup2;  // group 1 is sender, 2 is receiver
     rankInJoin    = src.rankInJoin;
     joinSize      = src.joinSize;
     compid1       = src.compid1;
     compid2       = src.compid2;
     comm_graph    = NULL;
     graph_type    = src.graph_type;
     context_id    = src.context_id;
     cover_set     = src.cover_set;
     return;
 }

References comm, comm_graph, compid1, compid2, context_id, cover_set, graph_type, joinSize, rankInGroup1, rankInGroup2, rankInJoin, receiverTasks, rootReceiver, rootSender, and senderTasks.

Member Function Documentation

◆ compute_partition()

ErrorCode moab::ParCommGraph::compute_partition	(	ParallelComm *	pco,
		Range &	owned,
		int	met
	)

Definition at line 1133 of file ParCommGraph.cpp.

 {
     // we are on a task on sender, and need to compute a new partition;
     // primary cells need to be distributed to nb receivers tasks
     // first, we will use graph partitioner, with zoltan;
     // in the graph that we need to build, the first layer of ghosts is needed;
     // can we avoid that ? For example, we can find out from each boundary edge/face what is the
     // other cell (on the other side), then form the global graph, and call zoltan in parallel met 1
     // would be a geometric partitioner, and met 2 would be a graph partitioner for method 1 we do
     // not need any ghost exchange
  
     // find first edges that are shared
     if( owned.empty() )
         return MB_SUCCESS;  // nothing to do? empty partition is not allowed, maybe we should return
                             // error?
     Core* mb = (Core*)pco->get_moab();
  
     double t1, t2, t3;
     t1               = MPI_Wtime();
     int primaryDim   = mb->dimension_from_handle( *owned.rbegin() );
     int interfaceDim = primaryDim - 1;  // should be 1 or 2
     Range sharedEdges;
     ErrorCode rval;
  
     std::vector< int > shprocs( MAX_SHARING_PROCS );
     std::vector< EntityHandle > shhandles( MAX_SHARING_PROCS );
  
     Tag gidTag = mb->globalId_tag();
     int np;
     unsigned char pstatus;
  
     std::multimap< int, int > extraGraphEdges;
     // std::map<int, int> adjCellsId;
     std::map< int, int > extraCellsProc;
     // if method is 2, no need to do the exchange for adjacent cells across partition boundary
     // these maps above will be empty for method 2 (geometry)
     if( 1 == met )
     {
         rval = pco->get_shared_entities( /*int other_proc*/ -1, sharedEdges, interfaceDim,
                                          /*const bool iface*/ true );MB_CHK_ERR( rval );
  
 #ifdef VERBOSE
         std::cout << " on sender task " << pco->rank() << " number of shared interface cells " << sharedEdges.size()
                   << "\n";
 #endif
         // find to what processors we need to send the ghost info about the edge
         // first determine the local graph; what elements are adjacent to each cell in owned range
         // cells that are sharing a partition interface edge, are identified first, and form a map
         TupleList TLe;                                     // tuple list for cells
         TLe.initialize( 2, 0, 1, 0, sharedEdges.size() );  // send to, id of adj cell, remote edge
         TLe.enableWriteAccess();
  
         std::map< EntityHandle, int > edgeToCell;  // from local boundary edge to adjacent cell id
         // will be changed after
         for( Range::iterator eit = sharedEdges.begin(); eit != sharedEdges.end(); eit++ )
         {
             EntityHandle edge = *eit;
             // get the adjacent cell
             Range adjEnts;
             rval = mb->get_adjacencies( &edge, 1, primaryDim, false, adjEnts );MB_CHK_ERR( rval );
             if( adjEnts.size() > 0 )
             {
                 EntityHandle adjCell = adjEnts[0];
                 int gid;
                 rval = mb->tag_get_data( gidTag, &adjCell, 1, &gid );MB_CHK_ERR( rval );
                 rval = pco->get_sharing_data( edge, shprocs.data() , shhandles.data() , pstatus, np );MB_CHK_ERR( rval );
                 int n                = TLe.get_n();
                 TLe.vi_wr[2 * n]     = shprocs[0];
                 TLe.vi_wr[2 * n + 1] = gid;
                 TLe.vul_wr[n]        = shhandles[0];  // the remote edge corresponding to shared edge
                 edgeToCell[edge]     = gid;           // store the map between edge and local id of adj cell
                 TLe.inc_n();
             }
         }
  
 #ifdef VERBOSE
         std::stringstream ff2;
         ff2 << "TLe_" << pco->rank() << ".txt";
         TLe.print_to_file( ff2.str().c_str() );
 #endif
         // send the data to the other processors:
         ( pco->proc_config().crystal_router() )->gs_transfer( 1, TLe, 0 );
         // on receiver side, each local edge will have the remote cell adjacent to it!
  
         int ne = TLe.get_n();
         for( int i = 0; i < ne; i++ )
         {
             int sharedProc         = TLe.vi_rd[2 * i];       // this info is coming from here, originally
             int remoteCellID       = TLe.vi_rd[2 * i + 1];   // this is the id of the remote cell, on sharedProc
             EntityHandle localCell = TLe.vul_rd[i];          // this is now local edge/face on this proc
             int localCellId        = edgeToCell[localCell];  // this is the local cell  adjacent to edge/face
             // now, we will need to add to the graph the pair <localCellId, remoteCellID>
             std::pair< int, int > extraAdj = std::make_pair( localCellId, remoteCellID );
             extraGraphEdges.insert( extraAdj );
             // adjCellsId [edgeToCell[localCell]] = remoteCellID;
             extraCellsProc[remoteCellID] = sharedProc;
 #ifdef VERBOSE
             std::cout << "local ID " << edgeToCell[localCell] << " remote cell ID: " << remoteCellID << "\n";
 #endif
         }
     }
     t2 = MPI_Wtime();
     if( rootSender ) std::cout << " time preparing the input for Zoltan:" << t2 - t1 << " seconds. \n";
         // so adj cells ids; need to call zoltan for parallel partition
 #ifdef MOAB_HAVE_ZOLTAN
     ZoltanPartitioner* mbZTool = new ZoltanPartitioner( mb, pco );
     if( 1 <= met )  //  partition in zoltan, either graph or geometric partitioner
     {
         std::map< int, Range > distribution;  // how to distribute owned elements by processors in receiving groups
         // in how many tasks do we want to be distributed?
         int numNewPartitions = (int)receiverTasks.size();
         Range primaryCells   = owned.subset_by_dimension( primaryDim );
         rval = mbZTool->partition_owned_cells( primaryCells, extraGraphEdges, extraCellsProc, numNewPartitions,
                                                distribution, met );MB_CHK_ERR( rval );
         for( std::map< int, Range >::iterator mit = distribution.begin(); mit != distribution.end(); mit++ )
         {
             int part_index = mit->first;
             assert( part_index < numNewPartitions );
             split_ranges[receiverTasks[part_index]] = mit->second;
         }
     }
     // delete now the partitioner
     delete mbZTool;
 #endif
     t3 = MPI_Wtime();
     if( rootSender ) std::cout << " time spent by Zoltan " << t3 - t2 << " seconds. \n";
     return MB_SUCCESS;
 }

References moab::Range::begin(), moab::ProcConfig::crystal_router(), moab::Range::empty(), moab::TupleList::enableWriteAccess(), moab::Range::end(), ErrorCode, moab::ParallelComm::get_moab(), moab::TupleList::get_n(), moab::ParallelComm::get_shared_entities(), moab::ParallelComm::get_sharing_data(), moab::TupleList::inc_n(), moab::TupleList::initialize(), MAX_SHARING_PROCS, mb, MB_CHK_ERR, MB_SUCCESS, ZoltanPartitioner::partition_owned_cells(), moab::TupleList::print_to_file(), moab::ParallelComm::proc_config(), moab::ParallelComm::rank(), moab::Range::rbegin(), receiverTasks, rootSender, moab::Range::size(), split_ranges, moab::Range::subset_by_dimension(), moab::TupleList::vi_rd, moab::TupleList::vi_wr, moab::TupleList::vul_rd, and moab::TupleList::vul_wr.

Referenced by iMOAB_SendMesh().

◆ compute_trivial_partition()

ErrorCode moab::ParCommGraph::compute_trivial_partition ( std::vector< int > & numElemsPerTaskInGroup1 )

Based on the number of elements on each task in group 1, partition for group 2, trivially.

Operations: it is called on every receiver task; decides how are all elements distributed

Note: establish how many elements are sent from each task in group 1 to tasks in group 2 This call is usually made on a root / master process, and will construct local maps that are member data, which contain the communication graph, in both directions Also, number of elements migrated/exchanged between each sender/receiver

Parameters

[in] numElemsPerTaskInGroup1 (std::vector<int> &) number of elements on each sender task

Definition at line 100 of file ParCommGraph.cpp.

 {
  
     recv_graph.clear();
     recv_sizes.clear();
     sender_graph.clear();
     sender_sizes.clear();
  
     if( numElemsPerTaskInGroup1.size() != senderTasks.size() )
         return MB_FAILURE;  // each sender has a number of elements that it owns
  
     // first find out total number of elements to be sent from all senders
     int total_elems = 0;
     std::vector< int > accum;
     accum.push_back( 0 );
  
     int num_senders = (int)senderTasks.size();
  
     for( size_t k = 0; k < numElemsPerTaskInGroup1.size(); k++ )
     {
         total_elems += numElemsPerTaskInGroup1[k];
         accum.push_back( total_elems );
     }
  
     int num_recv = ( (int)receiverTasks.size() );
     // in trivial partition, every receiver should get about total_elems/num_receivers elements
     int num_per_receiver = (int)( total_elems / num_recv );
     int leftover         = total_elems - num_per_receiver * num_recv;
  
     // so receiver k will receive  [starts[k], starts[k+1] ) interval
     std::vector< int > starts;
     starts.resize( num_recv + 1 );
     starts[0] = 0;
     for( int k = 0; k < num_recv; k++ )
     {
         starts[k + 1] = starts[k] + num_per_receiver;
         if( k < leftover ) starts[k + 1]++;
     }
  
     // each sender will send to a number of receivers, based on how the
     // arrays starts[0:num_recv] and accum[0:sendr] overlap
     int lastUsedReceiverRank = 0;  // first receiver was not treated yet
     for( int j = 0; j < num_senders; j++ )
     {
         // we could start the receiver loop with the latest receiver that received from previous
         // sender
         for( int k = lastUsedReceiverRank; k < num_recv; k++ )
         {
             // if overlap:
             if( starts[k] < accum[j + 1] && starts[k + 1] > accum[j] )
             {
                 recv_graph[receiverTasks[k]].push_back( senderTasks[j] );
                 sender_graph[senderTasks[j]].push_back( receiverTasks[k] );
  
                 // we still need to decide what is the overlap
                 int sizeOverlap = 1;  // at least 1, for sure
                 // 1
                 if( starts[k] >= accum[j] )  // one end is starts[k]
                 {
                     if( starts[k + 1] >= accum[j + 1] )  // the other end is accum[j+1]
                         sizeOverlap = accum[j + 1] - starts[k];
                     else  //
                         sizeOverlap = starts[k + 1] - starts[k];
                 }
                 else  // one end is accum[j]
                 {
                     if( starts[k + 1] >= accum[j + 1] )  // the other end is accum[j+1]
                         sizeOverlap = accum[j + 1] - accum[j];
                     else
                         sizeOverlap = starts[k + 1] - accum[j];
                 }
                 recv_sizes[receiverTasks[k]].push_back( sizeOverlap );  // basically, task k will receive from
                                                                         //   sender j, sizeOverlap elems
                 sender_sizes[senderTasks[j]].push_back( sizeOverlap );
                 if( starts[k] > accum[j + 1] )
                 {
                     lastUsedReceiverRank = k - 1;  // so next k loop will start a little higher, we
                                                    // probably finished with first few receivers (up
                                                    // to receiver lastUsedReceiverRank)
                     break;  // break the k loop, we distributed all elements from sender j to some
                             // receivers
                 }
             }
         }
     }
  
     return MB_SUCCESS;
 }

References MB_SUCCESS, receiverTasks, recv_graph, recv_sizes, sender_graph, sender_sizes, and senderTasks.

Referenced by iMOAB_SendMesh().

◆ dump_comm_information()

ErrorCode moab::ParCommGraph::dump_comm_information	(	std::string	prefix,
		int	is_send
	)

Definition at line 1365 of file ParCommGraph.cpp.

 {
     //
     if( -1 != rankInGroup1 && 1 == is_send )  // it is a sender task
     {
         std::ofstream dbfile;
         std::stringstream outf;
         outf << prefix << "_sender_" << rankInGroup1 << "_joint" << rankInJoin << "_type_" << (int)graph_type << ".txt";
         dbfile.open( outf.str().c_str() );
  
         if( graph_type == COVERAGE )
         {
             for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
                  mit != involved_IDs_map.end(); mit++ )
             {
                 int receiver_proc        = mit->first;
                 std::vector< int >& eids = mit->second;
                 dbfile << "receiver: " << receiver_proc << " size:" << eids.size() << "\n";
             }
         }
         else if( graph_type == INITIAL_MIGRATE )  // just after migration
         {
             for( std::map< int, Range >::iterator mit = split_ranges.begin(); mit != split_ranges.end(); mit++ )
             {
                 int receiver_proc = mit->first;
                 Range& eids       = mit->second;
                 dbfile << "receiver: " << receiver_proc << " size:" << eids.size() << "\n";
             }
         }
         else if( graph_type == DOF_BASED )  // just after migration, or from computeGraph
         {
             for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
                  mit != involved_IDs_map.end(); mit++ )
             {
                 int receiver_proc = mit->first;
                 dbfile << "receiver: " << receiver_proc << " size:" << mit->second.size() << "\n";
             }
         }
         dbfile.close();
     }
     if( -1 != rankInGroup2 && 0 == is_send )  // it is a receiver task
     {
         std::ofstream dbfile;
         std::stringstream outf;
         outf << prefix << "_receiver_" << rankInGroup2 << "_joint" << rankInJoin << "_type_" << (int)graph_type
              << ".txt";
         dbfile.open( outf.str().c_str() );
  
         if( graph_type == COVERAGE )
         {
             for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
                  mit != involved_IDs_map.end(); mit++ )
             {
                 int sender_proc          = mit->first;
                 std::vector< int >& eids = mit->second;
                 dbfile << "sender: " << sender_proc << " size:" << eids.size() << "\n";
             }
         }
         else if( graph_type == INITIAL_MIGRATE )  // just after migration
         {
             for( std::map< int, Range >::iterator mit = split_ranges.begin(); mit != split_ranges.end(); mit++ )
             {
                 int sender_proc = mit->first;
                 Range& eids     = mit->second;
                 dbfile << "sender: " << sender_proc << " size:" << eids.size() << "\n";
             }
         }
         else if( graph_type == DOF_BASED )  // just after migration
         {
             for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
                  mit != involved_IDs_map.end(); mit++ )
             {
                 int sender_proc = mit->first;
                 dbfile << "receiver: " << sender_proc << " size:" << mit->second.size() << "\n";
             }
         }
         dbfile.close();
     }
     return MB_SUCCESS;
 }

References COVERAGE, DOF_BASED, graph_type, INITIAL_MIGRATE, involved_IDs_map, MB_SUCCESS, rankInGroup1, rankInGroup2, rankInJoin, moab::Range::size(), and split_ranges.

◆ find_group_ranks()

void moab::ParCommGraph::find_group_ranks	(	MPI_Group	group,
		MPI_Comm	join,
		std::vector< int > &	ranks
	)

private

find ranks of a group with respect to an encompassing communicator

Operations: Local, usually called on root process of the group

Parameters

[in]	joincomm	(MPI_Comm)
[in]	group	(MPI_Group)
[out]	ranks	( std::vector<int>) ranks with respect to the joint communicator

Definition at line 81 of file ParCommGraph.cpp.

 {
     MPI_Group global_grp;
     MPI_Comm_group( joincomm, &global_grp );
  
     int grp_size;
  
     MPI_Group_size( group, &grp_size );
     std::vector< int > rks( grp_size );
     ranks.resize( grp_size );
  
     for( int i = 0; i < grp_size; i++ )
         rks[i] = i;
  
     MPI_Group_translate_ranks( group, grp_size, rks.data(), global_grp, ranks.data() );
     MPI_Group_free( &global_grp );
     return;
 }

Referenced by ParCommGraph().

◆ get_component_id1()

int moab::ParCommGraph::get_component_id1 ( )

inline

Definition at line 153 of file ParCommGraph.hpp.

     {
         return compid1;
     }

References compid1.

◆ get_component_id2()

int moab::ParCommGraph::get_component_id2 ( )

inline

Definition at line 157 of file ParCommGraph.hpp.

     {
         return compid2;
     }

References compid2.

◆ get_context_id()

int moab::ParCommGraph::get_context_id ( )

inline

Definition at line 162 of file ParCommGraph.hpp.

     {
         return context_id;
     }

References context_id.

◆ get_cover_set()

EntityHandle moab::ParCommGraph::get_cover_set ( )

inline

Definition at line 171 of file ParCommGraph.hpp.

     {
         return cover_set;
     }

References cover_set.

Referenced by iMOAB_ReceiveElementTag(), and iMOAB_SendElementTag().

◆ is_root_receiver()

bool moab::ParCommGraph::is_root_receiver ( )

inline

Definition at line 138 of file ParCommGraph.hpp.

     {
         return rootReceiver;
     }

References rootReceiver.

◆ is_root_sender()

bool moab::ParCommGraph::is_root_sender ( )

inline

Definition at line 133 of file ParCommGraph.hpp.

     {
         return rootSender;
     }

References rootSender.

Referenced by send_graph(), send_graph_partition(), and send_mesh_parts().

◆ pack_receivers_graph()

ErrorCode moab::ParCommGraph::pack_receivers_graph ( std::vector< int > & packed_recv_array )

pack information about receivers view of the graph, for future sending to receiver root

Operations: Local, called on root process of the senders group

Parameters

[out] packed_recv_array packed data will be sent to the root of receivers, and distributed from there, and will have this information, for each receiver, concatenated receiver 1 task, number of senders for receiver 1, then sender tasks for receiver 1, receiver 2 task, number of senders for receiver 2, sender tasks for receiver 2, etc Note: only the root of senders will compute this, and send it over to the receiver root, which will distribute it over each receiver; We do not pack the sizes of data to be sent, only the senders for each of the receivers (could be of size O(n^2) , where n is the number of tasks ; but in general, it should be more like O(n) ). Each sender sends to a "finite" number of receivers, and each receiver receives from a finite number of senders). We need this info to decide how to set up the send/receive waiting game for non-blocking communication )

Definition at line 189 of file ParCommGraph.cpp.

 {
     // it will basically look at local data, to pack communication graph, each receiver task will
     // have to post receives for each sender task that will send data to it; the array will be
     // communicated to root receiver, and eventually distributed to receiver tasks
  
     /*
      * packed_array will have receiver, number of senders, then senders, etc
      */
     if( recv_graph.size() < receiverTasks.size() )
     {
         // big problem, we have empty partitions in receive
         std::cout << " WARNING: empty partitions, some receiver tasks will receive nothing.\n";
     }
     for( std::map< int, std::vector< int > >::iterator it = recv_graph.begin(); it != recv_graph.end(); it++ )
     {
         int recv                    = it->first;
         std::vector< int >& senders = it->second;
         packed_recv_array.push_back( recv );
         packed_recv_array.push_back( (int)senders.size() );
  
         for( int k = 0; k < (int)senders.size(); k++ )
             packed_recv_array.push_back( senders[k] );
     }
  
     return MB_SUCCESS;
 }

References MB_SUCCESS, receiverTasks, recv_graph, and senders().

Referenced by send_graph().

◆ receive_comm_graph()

ErrorCode moab::ParCommGraph::receive_comm_graph	(	MPI_Comm	jcomm,
		ParallelComm *	pco,
		std::vector< int > &	pack_array
	)

Definition at line 358 of file ParCommGraph.cpp.

 {
     // first, receive from sender_rank 0, the communication graph (matrix), so each receiver
     // knows what data to expect
     MPI_Comm receive = pco->comm();
     int size_pack_array, ierr;
     MPI_Status status;
     if( rootReceiver )
     {
         /*
          * MPI_Probe(
         int source,
         int tag,
         MPI_Comm comm,
         MPI_Status* status)
          *
          */
         int mtag = compid2;
         ierr     = MPI_Probe( sender( 0 ), mtag, jcomm, &status );
         if( 0 != ierr )
         {
             std::cout << " MPI_Probe failure: " << ierr << "\n";
             return MB_FAILURE;
         }
         // get the count of data received from the MPI_Status structure
         ierr = MPI_Get_count( &status, MPI_INT, &size_pack_array );
         if( 0 != ierr )
         {
             std::cout << " MPI_Get_count failure: " << ierr << "\n";
             return MB_FAILURE;
         }
 #ifdef VERBOSE
         std::cout << " receive comm graph size: " << size_pack_array << "\n";
 #endif
         pack_array.resize( size_pack_array );
         ierr = MPI_Recv( pack_array.data(), size_pack_array, MPI_INT, sender( 0 ), mtag, jcomm, &status );
         if( 0 != ierr ) return MB_FAILURE;
 #ifdef VERBOSE
         std::cout << " receive comm graph ";
         for( int k = 0; k < (int)pack_array.size(); k++ )
             std::cout << " " << pack_array[k];
         std::cout << "\n";
 #endif
     }
  
     // now broadcast this whole array to all receivers, so they know what to expect
     ierr = MPI_Bcast( &size_pack_array, 1, MPI_INT, 0, receive );
     if( 0 != ierr ) return MB_FAILURE;
     pack_array.resize( size_pack_array );
     ierr = MPI_Bcast( pack_array.data(), size_pack_array, MPI_INT, 0, receive );
     if( 0 != ierr ) return MB_FAILURE;
     return MB_SUCCESS;
 }

References moab::ParallelComm::comm(), compid2, MB_SUCCESS, rootReceiver, and sender().

Referenced by iMOAB_ReceiveMesh().

◆ receive_mesh()

ErrorCode moab::ParCommGraph::receive_mesh	(	MPI_Comm	jcomm,
		ParallelComm *	pco,
		EntityHandle	local_set,
		std::vector< int > &	senders_local
	)

Definition at line 412 of file ParCommGraph.cpp.

 {
     ErrorCode rval;
     int ierr;
     MPI_Status status;
     // we also need to fill corresponding mesh info on the other side
     corr_tasks = senders_local;
     Range newEnts;
  
     Tag orgSendProcTag;  // this will be a tag set on the received mesh, with info about from what
                          // task / PE the
     // primary element came from, in the joint communicator ; this will be forwarded by coverage
     // mesh
     int defaultInt = -1;  // no processor, so it was not migrated from somewhere else
     rval           = pco->get_moab()->tag_get_handle( "orig_sending_processor", 1, MB_TYPE_INTEGER, orgSendProcTag,
                                                       MB_TAG_DENSE | MB_TAG_CREAT, &defaultInt );MB_CHK_SET_ERR( rval, "can't create original sending processor tag" );
     int mtag = compid2;
     if( !senders_local.empty() )
     {
         for( size_t k = 0; k < senders_local.size(); k++ )
         {
             int sender1 = senders_local[k];
             // first receive the size of the buffer using probe
             /*
                  * MPI_Probe(
                 int source,
                 int tag,
                 MPI_Comm comm,
                 MPI_Status* status)
                  *
                  */
             ierr = MPI_Probe( sender1, mtag, jcomm, &status );
             if( 0 != ierr )
             {
                 std::cout << " MPI_Probe failure in ParCommGraph::receive_mesh " << ierr << "\n";
                 return MB_FAILURE;
             }
             // get the count of data received from the MPI_Status structure
             int size_pack;
             ierr = MPI_Get_count( &status, MPI_CHAR, &size_pack );
             if( 0 != ierr )
             {
                 std::cout << " MPI_Get_count failure in ParCommGraph::receive_mesh  " << ierr << "\n";
                 return MB_FAILURE;
             }
  
             /* ierr = MPI_Recv (&size_pack, 1, MPI_INT, sender1, 1, jcomm, &status);
              if (0!=ierr) return MB_FAILURE;*/
             // now resize the buffer, then receive it
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_pack );
             // buffer->reserve(size_pack);
  
             ierr = MPI_Recv( buffer->mem_ptr, size_pack, MPI_UNSIGNED_CHAR, sender1, mtag, jcomm, &status );
             if( 0 != ierr )
             {
                 std::cout << " MPI_Recv failure in ParCommGraph::receive_mesh " << ierr << "\n";
                 return MB_FAILURE;
             }
             // now unpack the buffer we just received
             Range entities;
             std::vector< std::vector< EntityHandle > > L1hloc, L1hrem;
             std::vector< std::vector< int > > L1p;
             std::vector< EntityHandle > L2hloc, L2hrem;
             std::vector< unsigned int > L2p;
  
             buffer->reset_ptr( sizeof( int ) );
             std::vector< EntityHandle > entities_vec( entities.size() );
             std::copy( entities.begin(), entities.end(), entities_vec.begin() );
             rval = pco->unpack_buffer( buffer->buff_ptr, false, -1, -1, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p,
                                        entities_vec );
             delete buffer;
             if( MB_SUCCESS != rval ) return rval;
  
             std::copy( entities_vec.begin(), entities_vec.end(), range_inserter( entities ) );
             // we have to add them to the local set
             rval = pco->get_moab()->add_entities( local_set, entities );
             if( MB_SUCCESS != rval ) return rval;
             // corr_sizes is the size of primary entities received
             Range verts              = entities.subset_by_dimension( 0 );
             Range local_primary_ents = subtract( entities, verts );
             if( local_primary_ents.empty() )
             {
                 // it is possible that all ents sent were vertices (point cloud)
                 // then consider primary entities the vertices
                 local_primary_ents = verts;
             }
             else
             {
                 // set a tag with the original sender for the primary entity
                 // will be used later for coverage mesh
                 std::vector< int > orig_senders( local_primary_ents.size(), sender1 );
                 rval = pco->get_moab()->tag_set_data( orgSendProcTag, local_primary_ents, orig_senders.data() );
             }
             corr_sizes.push_back( (int)local_primary_ents.size() );
  
             newEnts.merge( entities );
             // make these in split ranges
             split_ranges[sender1] = local_primary_ents;
  
 #ifdef VERBOSE
             std::ostringstream partial_outFile;
  
             partial_outFile << "part_send_" << sender1 << "."
                             << "recv" << rankInJoin << ".vtk";
  
             // the mesh contains ghosts too, but they are not part of mat/neumann set
             // write in serial the file, to see what tags are missing
             std::cout << " writing from receiver " << rankInJoin << " from sender " << sender1
                       << " entities: " << entities.size() << std::endl;
             rval = pco->get_moab()->write_file( partial_outFile.str().c_str(), 0, 0, &local_set,
                                                 1 );  // everything on local set received
             if( MB_SUCCESS != rval ) return rval;
 #endif
         }
     }
     // make sure adjacencies are updated on the new elements
  
     if( newEnts.empty() )
     {
         std::cout << " WARNING: this task did not receive any entities \n";
     }
     // in order for the merging to work, we need to be sure that the adjacencies are updated
     // (created)
     Range local_verts        = newEnts.subset_by_type( MBVERTEX );
     newEnts                  = subtract( newEnts, local_verts );
     Core* mb                 = (Core*)pco->get_moab();
     AEntityFactory* adj_fact = mb->a_entity_factory();
     if( !adj_fact->vert_elem_adjacencies() )
         adj_fact->create_vert_elem_adjacencies();
     else
     {
         for( Range::iterator it = newEnts.begin(); it != newEnts.end(); ++it )
         {
             EntityHandle eh          = *it;
             const EntityHandle* conn = NULL;
             int num_nodes            = 0;
             rval                     = mb->get_connectivity( eh, conn, num_nodes );MB_CHK_ERR( rval );
             adj_fact->notify_create_entity( eh, conn, num_nodes );
         }
     }
  
     return MB_SUCCESS;
 }

References moab::Interface::add_entities(), moab::Range::begin(), buffer, compid2, corr_sizes, corr_tasks, moab::AEntityFactory::create_vert_elem_adjacencies(), moab::Range::empty(), moab::Range::end(), entities, ErrorCode, moab::ParallelComm::get_moab(), mb, MB_CHK_ERR, MB_CHK_SET_ERR, MB_SUCCESS, MB_TAG_CREAT, MB_TAG_DENSE, MB_TYPE_INTEGER, MBVERTEX, moab::Range::merge(), moab::AEntityFactory::notify_create_entity(), rankInJoin, moab::Range::size(), split_ranges, moab::Range::subset_by_type(), moab::subtract(), moab::Interface::tag_get_handle(), moab::Interface::tag_set_data(), moab::ParallelComm::unpack_buffer(), moab::AEntityFactory::vert_elem_adjacencies(), and moab::Interface::write_file().

Referenced by iMOAB_ReceiveMesh().

◆ receive_tag_values()

ErrorCode moab::ParCommGraph::receive_tag_values	(	MPI_Comm	jcomm,
		ParallelComm *	pco,
		Range &	owned,
		std::vector< Tag > &	tag_handles
	)

Get the size of the specified tag in bytes

Definition at line 786 of file ParCommGraph.cpp.

 {
     // opposite to sending, we will use blocking receives
     int ierr;
     MPI_Status status;
     // basically, owned.size() needs to be equal to sum(corr_sizes)
     // get info about the tag size, type, etc
     Core* mb = (Core*)pco->get_moab();
     // get info about the tag
     //! Get the size of the specified tag in bytes
     ErrorCode rval;
     int total_bytes_per_entity = 0;
     std::vector< int > vect_bytes_per_tag;
 #ifdef VERBOSE
     std::vector< int > tag_sizes;
 #endif
     for( size_t i = 0; i < tag_handles.size(); i++ )
     {
         int bytes_per_tag;
         rval = mb->tag_get_bytes( tag_handles[i], bytes_per_tag );MB_CHK_ERR( rval );
         total_bytes_per_entity += bytes_per_tag;
         vect_bytes_per_tag.push_back( bytes_per_tag );
 #ifdef VERBOSE
         int tag_size;
         rval = mb->tag_get_length( tag_handles[i], tag_size );MB_CHK_ERR( rval );
         tag_sizes.push_back( tag_size );
 #endif
     }
  
     int mtag = compid1 + compid2;
  
     if( graph_type == INITIAL_MIGRATE )
     {
         // std::map<int, Range> split_ranges;
         // rval = split_owned_range ( owned);MB_CHK_ERR ( rval );
  
         // use the buffers data structure to allocate memory for receiving the tags
         for( std::map< int, Range >::iterator it = split_ranges.begin(); it != split_ranges.end(); it++ )
         {
             int sender_proc = it->first;
             Range ents      = it->second;  // primary entities, with the tag data, we will receive
             int size_buffer = 4 + total_bytes_per_entity *
                                       (int)ents.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
  
             buffer->reset_ptr( sizeof( int ) );
  
             *( (int*)buffer->mem_ptr ) = size_buffer;
             // int size_pack = buffer->get_current_size(); // debug check
  
             ierr = MPI_Recv( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, sender_proc, mtag, jcomm, &status );
             if( ierr != 0 ) return MB_FAILURE;
             // now set the tag
             // copy to tag
  
             for( size_t i = 0; i < tag_handles.size(); i++ )
             {
                 rval = mb->tag_set_data( tag_handles[i], ents, (void*)( buffer->buff_ptr ) );
                 buffer->buff_ptr += vect_bytes_per_tag[i] * ents.size();
             }
             delete buffer;  // no need for it afterwards
             MB_CHK_ERR( rval );
         }
     }
     else if( graph_type == COVERAGE )  // receive buffer, then extract tag data, in a loop
     {
         // we know that we will need to receive some tag data in a specific order (by ids stored)
         // first, get the ids of the local elements, from owned Range; unpack the buffer in order
         Tag gidTag = mb->globalId_tag();
         std::vector< int > gids;
         gids.resize( owned.size() );
         rval = mb->tag_get_data( gidTag, owned, gids.data() );MB_CHK_ERR( rval );
         std::map< int, EntityHandle > gidToHandle;
         size_t i = 0;
         for( Range::iterator it = owned.begin(); it != owned.end(); it++ )
         {
             EntityHandle eh        = *it;
             gidToHandle[gids[i++]] = eh;
         }
         //
         // now, unpack the data and set it to the tag
         for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
              mit != involved_IDs_map.end(); mit++ )
         {
             int sender_proc          = mit->first;
             std::vector< int >& eids = mit->second;
             int size_buffer          = 4 + total_bytes_per_entity *
                                       (int)eids.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
             buffer->reset_ptr( sizeof( int ) );
             *( (int*)buffer->mem_ptr ) = size_buffer;  // this is really not necessary, it should receive this too
  
             // receive the buffer
             ierr = MPI_Recv( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, sender_proc, mtag, jcomm, &status );
             if( ierr != 0 ) return MB_FAILURE;
 // start copy
 #ifdef VERBOSE
             std::ofstream dbfile;
             std::stringstream outf;
             outf << "recvFrom_" << sender_proc << "_on_proc_" << rankInJoin << ".txt";
             dbfile.open( outf.str().c_str() );
             dbfile << "recvFrom_" << sender_proc << " on proc  " << rankInJoin << "\n";
 #endif
  
             // copy tag data from buffer->buff_ptr
             // data is arranged by tag , and repeat the loop for each entity ()
             // maybe it should be arranged by entity now, not by tag (so one loop for entities,
             // outside)
  
             for( std::vector< int >::iterator it = eids.begin(); it != eids.end(); ++it )
             {
                 int eID                                      = *it;
                 std::map< int, EntityHandle >::iterator mit2 = gidToHandle.find( eID );
                 if( mit2 == gidToHandle.end() )
                 {
                     std::cout << " on rank: " << rankInJoin << " cannot find entity handle with global ID " << eID
                               << "\n";
                     return MB_FAILURE;
                 }
                 EntityHandle eh = mit2->second;
                 for( i = 0; i < tag_handles.size(); i++ )
                 {
                     rval = mb->tag_set_data( tag_handles[i], &eh, 1, (void*)( buffer->buff_ptr ) );MB_CHK_ERR( rval );
 #ifdef VERBOSE
                     dbfile << "global ID " << eID << " local handle " << mb->id_from_handle( eh ) << " vals: ";
                     double* vals = (double*)( buffer->buff_ptr );
                     for( int kk = 0; kk < tag_sizes[i]; kk++ )
                     {
                         dbfile << " " << *vals;
                         vals++;
                     }
                     dbfile << "\n";
 #endif
                     buffer->buff_ptr += vect_bytes_per_tag[i];
                 }
             }
  
             // delete receive buffer
             delete buffer;
 #ifdef VERBOSE
             dbfile.close();
 #endif
         }
     }
     else if( graph_type == DOF_BASED )
     {
         // need to fill up the values for each tag, in the order desired, from the buffer received
         //
         // get all the tags, for all owned entities, and pack the buffers accordingly
         // we do not want to get the tags by entity, it may be too expensive
         std::vector< std::vector< double > > valuesTags;
         valuesTags.resize( tag_handles.size() );
         for( size_t i = 0; i < tag_handles.size(); i++ )
         {
             int bytes_per_tag;
             rval = mb->tag_get_bytes( tag_handles[i], bytes_per_tag );MB_CHK_ERR( rval );
             valuesTags[i].resize( owned.size() * bytes_per_tag / sizeof( double ) );
             // fill the whole array, we will pick up from here
             // we will fill this array, using data from received buffer
             // rval = mb->tag_get_data(owned, (void*)( valuesTags[i].data() ) );MB_CHK_ERR ( rval );
         }
         // now, unpack the data and set the tags
         sendReqs.resize( involved_IDs_map.size() );
         for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
              mit != involved_IDs_map.end(); ++mit )
         {
             int sender_proc                     = mit->first;
             std::vector< int >& eids            = mit->second;
             std::vector< int >& index_in_values = map_index[sender_proc];
             std::vector< int >& index_ptr       = map_ptr[sender_proc];  // this is eids.size()+1;
             int size_buffer                     = 4 + total_bytes_per_entity *
                                       (int)eids.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
             buffer->reset_ptr( sizeof( int ) );
  
             // receive the buffer
             ierr = MPI_Recv( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, sender_proc, mtag, jcomm, &status );
             if( ierr != 0 ) return MB_FAILURE;
             // use the values in buffer to populate valuesTag arrays, fill it up!
             int j = 0;
             for( std::vector< int >::iterator it = eids.begin(); it != eids.end(); ++it, ++j )
             {
                 for( size_t i = 0; i < tag_handles.size(); i++ )
                 {
                     // right now, move just doubles; but it could be any type of tag
                     double val = *( (double*)( buffer->buff_ptr ) );
                     buffer->buff_ptr += 8;  // we know we are working with doubles only !!!
                     for( int k = index_ptr[j]; k < index_ptr[j + 1]; k++ )
                         valuesTags[i][index_in_values[k]] = val;
                 }
             }
             // we are done with the buffer in which we received tags, release / delete it
             delete buffer;
         }
         // now we populated the values for all tags; set now the tags!
         for( size_t i = 0; i < tag_handles.size(); i++ )
         {
             // we will fill this array, using data from received buffer
             rval = mb->tag_set_data( tag_handles[i], owned, (void*)( valuesTags[i].data() ) );MB_CHK_ERR( rval );
         }
     }
     return MB_SUCCESS;
 }

References moab::Range::begin(), buffer, compid1, compid2, COVERAGE, DOF_BASED, moab::Range::end(), ErrorCode, moab::ParallelComm::get_moab(), graph_type, INITIAL_MIGRATE, involved_IDs_map, map_index, map_ptr, mb, MB_CHK_ERR, MB_SUCCESS, rankInJoin, sendReqs, moab::Range::size(), and split_ranges.

Referenced by iMOAB_ReceiveElementTag().

◆ receiver()

int moab::ParCommGraph::receiver ( int index )

inline

Definition at line 148 of file ParCommGraph.hpp.

     {
         return receiverTasks[index];
     }

References receiverTasks.

Referenced by iMOAB_ReceiveMesh(), and send_graph().

◆ receivers()

const std::vector< int >& moab::ParCommGraph::receivers ( )

inline

Definition at line 210 of file ParCommGraph.hpp.

     {
         return receiverTasks;
     }

References receiverTasks.

Referenced by split_owned_range().

◆ release_send_buffers()

ErrorCode moab::ParCommGraph::release_send_buffers ( )

Definition at line 560 of file ParCommGraph.cpp.

 {
     int ierr, nsize = (int)sendReqs.size();
     std::vector< MPI_Status > mult_status;
     mult_status.resize( sendReqs.size() );
     ierr = MPI_Waitall( nsize, sendReqs.data(), mult_status.data() );
  
     if( ierr != 0 ) return MB_FAILURE;
     // now we can free all buffers
     delete[] comm_graph;
     comm_graph = NULL;
     std::vector< ParallelComm::Buffer* >::iterator vit;
     for( vit = localSendBuffs.begin(); vit != localSendBuffs.end(); ++vit )
         delete( *vit );
     localSendBuffs.clear();
     return MB_SUCCESS;
 }

References comm_graph, localSendBuffs, MB_SUCCESS, and sendReqs.

◆ send_graph()

ErrorCode moab::ParCommGraph::send_graph ( MPI_Comm jcomm )

use tag 10 to send size and tag 20 to send the packed array

Definition at line 265 of file ParCommGraph.cpp.

 {
     if( is_root_sender() )
     {
         int ierr;
         // will need to build a communication graph, because each sender knows now to which receiver
         // to send data the receivers need to post receives for each sender that will send data to
         // them will need to gather on rank 0 on the sender comm, global ranks of sender with
         // receivers to send build communication matrix, each receiver will receive from what sender
  
         std::vector< int > packed_recv_array;
         ErrorCode rval = pack_receivers_graph( packed_recv_array );
         if( MB_SUCCESS != rval ) return rval;
  
         int size_pack_array = (int)packed_recv_array.size();
         comm_graph          = new int[size_pack_array + 1]; // this should be at least size 2
         comm_graph[0]       = size_pack_array;
         for( int k = 0; k < size_pack_array; k++ )
             comm_graph[k + 1] = packed_recv_array[k];
         // will add 2 requests
         /// use tag 10 to send size and tag 20 to send the packed array
         sendReqs.resize( 1 );
         // do not send the size in advance, because we use probe now
         /*ierr = MPI_Isend( comm_graph.data(), 1, MPI_INT, receiver(0), 10, jcomm, sendReqs.data()); // we
         have to use global communicator if (ierr!=0) return MB_FAILURE;*/
         int mtag = compid2;
         ierr     = MPI_Isend( &comm_graph[1], size_pack_array, MPI_INT, receiver( 0 ), mtag, jcomm,
                               sendReqs.data() );  // we have to use global communicator
         if( ierr != 0 ) return MB_FAILURE;
     }
     return MB_SUCCESS;
 }

References comm_graph, compid2, ErrorCode, is_root_sender(), MB_SUCCESS, pack_receivers_graph(), receiver(), and sendReqs.

Referenced by iMOAB_SendMesh(), and send_graph_partition().

◆ send_graph_partition()

ErrorCode moab::ParCommGraph::send_graph_partition	(	ParallelComm *	pco,
		MPI_Comm	jcomm
	)

Definition at line 1264 of file ParCommGraph.cpp.

 {
     // first, accumulate the info to root of sender; use gatherv
     // first, accumulate number of receivers from each sender, to the root receiver
     int numberReceivers =
         (int)split_ranges.size();            // these are ranges of elements to be sent to each receiver, from this task
     int nSenders = (int)senderTasks.size();  //
     // this sender will have to send to this many receivers
     std::vector< int > displs( 1 );  // displacements for gatherv
     std::vector< int > counts( 1 );
     if( is_root_sender() )
     {
         displs.resize( nSenders + 1 );
         counts.resize( nSenders );
     }
  
     int ierr = MPI_Gather( &numberReceivers, 1, MPI_INT, counts.data(), 1, MPI_INT, 0, pco->comm() );
     if( ierr != MPI_SUCCESS ) return MB_FAILURE;
     // compute now displacements
     if( is_root_sender() )
     {
         displs[0] = 0;
         for( int k = 0; k < nSenders; k++ )
         {
             displs[k + 1] = displs[k] + counts[k];
         }
     }
     std::vector< int > buffer;
     if( is_root_sender() ) buffer.resize( displs[nSenders] );  // the last one will have the total count now
  
     std::vector< int > recvs;
     for( std::map< int, Range >::iterator mit = split_ranges.begin(); mit != split_ranges.end(); mit++ )
     {
         recvs.push_back( mit->first );
     }
     ierr =
         MPI_Gatherv( recvs.data(), numberReceivers, MPI_INT, buffer.data(), counts.data(), displs.data(), MPI_INT, 0, pco->comm() );
     if( ierr != MPI_SUCCESS ) return MB_FAILURE;
  
     // now form recv_graph map; points from the
     // now form the graph to be sent to the other side; only on root
     if( is_root_sender() )
     {
 #ifdef GRAPH_INFO
         std::ofstream dbfileSender;
         std::stringstream outf;
         outf << "S_" << compid1 << "_R_" << compid2 << "_SenderGraph.txt";
         dbfileSender.open( outf.str().c_str() );
         dbfileSender << " number senders: " << nSenders << "\n";
         dbfileSender << " senderRank \treceivers \n";
         for( int k = 0; k < nSenders; k++ )
         {
             int indexInBuff = displs[k];
             int senderTask  = senderTasks[k];
             dbfileSender << senderTask << "\t\t";
             for( int j = 0; j < counts[k]; j++ )
             {
                 int recvTask = buffer[indexInBuff + j];
                 dbfileSender << recvTask << " ";
             }
             dbfileSender << "\n";
         }
         dbfileSender.close();
 #endif
         for( int k = 0; k < nSenders; k++ )
         {
             int indexInBuff = displs[k];
             int senderTask  = senderTasks[k];
             for( int j = 0; j < counts[k]; j++ )
             {
                 int recvTask = buffer[indexInBuff + j];
                 recv_graph[recvTask].push_back( senderTask );  // this will be packed and sent to root receiver, with
                                                                // nonblocking send
             }
         }
 #ifdef GRAPH_INFO
         std::ofstream dbfile;
         std::stringstream outf2;
         outf2 << "S_" << compid1 << "_R_" << compid2 << "_RecvGraph.txt";
         dbfile.open( outf2.str().c_str() );
         dbfile << " number receivers: " << recv_graph.size() << "\n";
         dbfile << " receiverRank \tsenders \n";
         for( std::map< int, std::vector< int > >::iterator mit = recv_graph.begin(); mit != recv_graph.end(); mit++ )
         {
             int recvTask                = mit->first;
             std::vector< int >& senders = mit->second;
             dbfile << recvTask << "\t\t";
             for( std::vector< int >::iterator vit = senders.begin(); vit != senders.end(); vit++ )
                 dbfile << *vit << " ";
             dbfile << "\n";
         }
         dbfile.close();
 #endif
         // this is the same as trivial partition
         ErrorCode rval = send_graph( jcomm );MB_CHK_ERR( rval );
     }
  
     return MB_SUCCESS;
 }

References buffer, moab::ParallelComm::comm(), compid1, compid2, ErrorCode, is_root_sender(), MB_CHK_ERR, MB_SUCCESS, recv_graph, send_graph(), senders(), senderTasks, and split_ranges.

Referenced by iMOAB_SendMesh().

◆ send_mesh_parts()

ErrorCode moab::ParCommGraph::send_mesh_parts	(	MPI_Comm	jcomm,
		ParallelComm *	pco,
		Range &	owned
	)

not anymore !

Definition at line 300 of file ParCommGraph.cpp.

 {
  
     ErrorCode rval;
     if( split_ranges.empty() )  // in trivial partition
     {
         rval = split_owned_range( rankInGroup1, owned );
         if( rval != MB_SUCCESS ) return rval;
         // we know this on the sender side:
         corr_tasks = sender_graph[senderTasks[rankInGroup1]];  // copy
         corr_sizes = sender_sizes[senderTasks[rankInGroup1]];  // another copy
     }
     int mtag     = compid2;
     int indexReq = 0;
     int ierr;                             // MPI error
     if( is_root_sender() ) indexReq = 1;  // for sendReqs
     sendReqs.resize( indexReq + split_ranges.size() );
     for( std::map< int, Range >::iterator it = split_ranges.begin(); it != split_ranges.end(); it++ )
     {
         int receiver_proc = it->first;
         Range ents        = it->second;
  
         // add necessary vertices too
         Range verts;
         rval = pco->get_moab()->get_adjacencies( ents, 0, false, verts, Interface::UNION );
         if( rval != MB_SUCCESS )
         {
             std::cout << " can't get adjacencies. for entities to send\n";
             return rval;
         }
         ents.merge( verts );
         ParallelComm::Buffer* buffer = new ParallelComm::Buffer( ParallelComm::INITIAL_BUFF_SIZE );
         buffer->reset_ptr( sizeof( int ) );
         rval = pco->pack_buffer( ents, false, true, false, -1, buffer );
         if( rval != MB_SUCCESS )
         {
             std::cout << " can't pack buffer for entities to send\n";
             return rval;
         }
         int size_pack = buffer->get_current_size();
  
         // TODO there could be an issue with endian things; check !!!!!
         // we are sending the size of the buffer first as an int!!!
         /// not anymore !
         /* ierr = MPI_Isend(buffer->mem_ptr, 1, MPI_INT, receiver_proc, 1, jcomm,
          &sendReqs[indexReq]); // we have to use global communicator if (ierr!=0) return MB_FAILURE;
          indexReq++;*/
  
         ierr = MPI_Isend( buffer->mem_ptr, size_pack, MPI_UNSIGNED_CHAR, receiver_proc, mtag, jcomm,
                           &sendReqs[indexReq] );  // we have to use global communicator
         if( ierr != 0 ) return MB_FAILURE;
         indexReq++;
         localSendBuffs.push_back( buffer );
     }
     return MB_SUCCESS;
 }

References buffer, compid2, corr_sizes, corr_tasks, ErrorCode, moab::Interface::get_adjacencies(), moab::ParallelComm::get_moab(), moab::ParallelComm::INITIAL_BUFF_SIZE, is_root_sender(), localSendBuffs, MB_SUCCESS, moab::Range::merge(), moab::ParallelComm::pack_buffer(), rankInGroup1, sender_graph, sender_sizes, senderTasks, sendReqs, split_owned_range(), split_ranges, and moab::Interface::UNION.

Referenced by iMOAB_SendMesh().

◆ send_tag_values()

ErrorCode moab::ParCommGraph::send_tag_values	(	MPI_Comm	jcomm,
		ParallelComm *	pco,
		Range &	owned,
		std::vector< Tag > &	tag_handles
	)

Get the size of the specified tag in bytes

Definition at line 580 of file ParCommGraph.cpp.

 {
     // basically, owned.size() needs to be equal to sum(corr_sizes)
     // get info about the tag size, type, etc
     int ierr;
     Core* mb = (Core*)pco->get_moab();
     // get info about the tag
     //! Get the size of the specified tag in bytes
     int total_bytes_per_entity = 0;  // we need to know, to allocate buffers
     ErrorCode rval;
     std::vector< int > vect_bytes_per_tag;
 #ifdef VERBOSE
     std::vector< int > tag_sizes;
 #endif
     for( size_t i = 0; i < tag_handles.size(); i++ )
     {
         int bytes_per_tag;
         rval = mb->tag_get_bytes( tag_handles[i], bytes_per_tag );MB_CHK_ERR( rval );
         int tag_size1;  // length
         rval = mb->tag_get_length( tag_handles[i], tag_size1 );MB_CHK_ERR( rval );
         if( graph_type == DOF_BASED )
             bytes_per_tag = bytes_per_tag / tag_size1;  // we know we have one double per tag , per ID sent;
                                                         // could be 8 for double, 4 for int, etc
         total_bytes_per_entity += bytes_per_tag;
         vect_bytes_per_tag.push_back( bytes_per_tag );
 #ifdef VERBOSE
         int tag_size;
         rval = mb->tag_get_length( tag_handles[i], tag_size );MB_CHK_ERR( rval );
         tag_sizes.push_back( tag_size );
 #endif
     }
  
     int mtag     = compid1 + compid2;  // used as mpi tag to differentiate a little the messages
     int indexReq = 0;
     if( graph_type == INITIAL_MIGRATE )  // original send
     {
         // use the buffers data structure to allocate memory for sending the tags
         sendReqs.resize( split_ranges.size() );
  
         for( std::map< int, Range >::iterator it = split_ranges.begin(); it != split_ranges.end(); it++ )
         {
             int receiver_proc = it->first;
             Range ents        = it->second;  // primary entities, with the tag data
             int size_buffer   = 4 + total_bytes_per_entity *
                                       (int)ents.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
  
             buffer->reset_ptr( sizeof( int ) );
             for( size_t i = 0; i < tag_handles.size(); i++ )
             {
                 // copy tag data to buffer->buff_ptr, and send the buffer (we could have used
                 // regular char arrays)
                 rval = mb->tag_get_data( tag_handles[i], ents, (void*)( buffer->buff_ptr ) );MB_CHK_ERR( rval );
                 // advance the butter
                 buffer->buff_ptr += vect_bytes_per_tag[i] * ents.size();
             }
             *( (int*)buffer->mem_ptr ) = size_buffer;
             // int size_pack = buffer->get_current_size(); // debug check
  
             ierr = MPI_Isend( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, receiver_proc, mtag, jcomm,
                               &sendReqs[indexReq] );  // we have to use global communicator
             if( ierr != 0 ) return MB_FAILURE;
             indexReq++;
             localSendBuffs.push_back( buffer );  // we will release them after nonblocking sends are completed
         }
     }
     else if( graph_type == COVERAGE )
     {
         // we know that we will need to send some tag data in a specific order
         // first, get the ids of the local elements, from owned Range; arrange the buffer in order
         // of increasing global id
         Tag gidTag = mb->globalId_tag();
         std::vector< int > gids;
         gids.resize( owned.size() );
         rval = mb->tag_get_data( gidTag, owned, gids.data() );MB_CHK_ERR( rval );
         std::map< int, EntityHandle > gidToHandle;
         size_t i = 0;
         for( Range::iterator it = owned.begin(); it != owned.end(); it++ )
         {
             EntityHandle eh        = *it;
             gidToHandle[gids[i++]] = eh;
         }
         // now, pack the data and send it
         sendReqs.resize( involved_IDs_map.size() );
         for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
              mit != involved_IDs_map.end(); mit++ )
         {
             int receiver_proc        = mit->first;
             std::vector< int >& eids = mit->second;
             int size_buffer          = 4 + total_bytes_per_entity *
                                       (int)eids.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
             buffer->reset_ptr( sizeof( int ) );
 #ifdef VERBOSE
             std::ofstream dbfile;
             std::stringstream outf;
             outf << "from_" << rankInJoin << "_send_to_" << receiver_proc << ".txt";
             dbfile.open( outf.str().c_str() );
             dbfile << "from " << rankInJoin << " send to " << receiver_proc << "\n";
 #endif
             // copy tag data to buffer->buff_ptr, and send the buffer (we could have used regular
             // char arrays) pack data by tag, to be consistent with above, even though we loop
             // through the entities for each tag
  
             for( std::vector< int >::iterator it = eids.begin(); it != eids.end(); it++ )
             {
                 int eID         = *it;
                 EntityHandle eh = gidToHandle[eID];
                 for( i = 0; i < tag_handles.size(); i++ )
                 {
                     rval = mb->tag_get_data( tag_handles[i], &eh, 1, (void*)( buffer->buff_ptr ) );
                     if( rval != MB_SUCCESS )
                     {
                         delete buffer;  // free parallel comm buffer first
  
                         MB_SET_ERR( rval, "Tag get data failed" );
                     }
 #ifdef VERBOSE
                     dbfile << "global ID " << eID << " local handle " << mb->id_from_handle( eh ) << " vals: ";
                     double* vals = (double*)( buffer->buff_ptr );
                     for( int kk = 0; kk < tag_sizes[i]; kk++ )
                     {
                         dbfile << " " << *vals;
                         vals++;
                     }
                     dbfile << "\n";
 #endif
                     buffer->buff_ptr += vect_bytes_per_tag[i];
                 }
             }
  
 #ifdef VERBOSE
             dbfile.close();
 #endif
             *( (int*)buffer->mem_ptr ) = size_buffer;
             // int size_pack = buffer->get_current_size(); // debug check
             ierr = MPI_Isend( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, receiver_proc, mtag, jcomm,
                               &sendReqs[indexReq] );  // we have to use global communicator
             if( ierr != 0 ) return MB_FAILURE;
             indexReq++;
             localSendBuffs.push_back( buffer );  // we will release them after nonblocking sends are completed
         }
     }
     else if( graph_type == DOF_BASED )
     {
         // need to fill up the buffer, in the order desired, send it
         // get all the tags, for all owned entities, and pack the buffers accordingly
         // we do not want to get the tags by entity, it may be too expensive
         std::vector< std::vector< double > > valuesTags;
         valuesTags.resize( tag_handles.size() );
         for( size_t i = 0; i < tag_handles.size(); i++ )
         {
             int bytes_per_tag;
             rval = mb->tag_get_bytes( tag_handles[i], bytes_per_tag );MB_CHK_ERR( rval );
             valuesTags[i].resize( owned.size() * bytes_per_tag / sizeof( double ) );
             // fill the whole array, we will pick up from here
             rval = mb->tag_get_data( tag_handles[i], owned, (void*)( valuesTags[i].data() ) );MB_CHK_ERR( rval );
         }
         // now, pack the data and send it
         sendReqs.resize( involved_IDs_map.size() );
         for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin();
              mit != involved_IDs_map.end(); ++mit )
         {
             int receiver_proc                   = mit->first;
             std::vector< int >& eids            = mit->second;
             std::vector< int >& index_in_values = map_index[receiver_proc];
             std::vector< int >& index_ptr       = map_ptr[receiver_proc];  // this is eids.size()+1;
             int size_buffer                     = 4 + total_bytes_per_entity *
                                       (int)eids.size();  // hopefully, below 2B; if more, we have a big problem ...
             ParallelComm::Buffer* buffer = new ParallelComm::Buffer( size_buffer );
             buffer->reset_ptr( sizeof( int ) );
 #ifdef VERBOSE
             std::ofstream dbfile;
             std::stringstream outf;
             outf << "from_" << rankInJoin << "_send_to_" << receiver_proc << ".txt";
             dbfile.open( outf.str().c_str() );
             dbfile << "from " << rankInJoin << " send to " << receiver_proc << "\n";
 #endif
             // copy tag data to buffer->buff_ptr, and send the buffer
             // pack data by tag, to be consistent with above
             int j = 0;
             for( std::vector< int >::iterator it = eids.begin(); it != eids.end(); it++, j++ )
             {
                 int index_in_v = index_in_values[index_ptr[j]];
                 for( size_t i = 0; i < tag_handles.size(); i++ )
                 {
                     // right now, move just doubles; but it could be any type of tag
                     *( (double*)( buffer->buff_ptr ) ) = valuesTags[i][index_in_v];
                     buffer->buff_ptr += 8;  // we know we are working with doubles only !!!
                 }
             };
             *( (int*)buffer->mem_ptr ) = size_buffer;
             // int size_pack = buffer->get_current_size(); // debug check
             ierr = MPI_Isend( buffer->mem_ptr, size_buffer, MPI_UNSIGNED_CHAR, receiver_proc, mtag, jcomm,
                               &sendReqs[indexReq] );  // we have to use global communicator
             if( ierr != 0 ) return MB_FAILURE;
             indexReq++;
             localSendBuffs.push_back( buffer );  // we will release them after nonblocking sends are completed
         }
     }
     return MB_SUCCESS;
 }

References moab::Range::begin(), buffer, compid1, compid2, COVERAGE, DOF_BASED, moab::Range::end(), ErrorCode, moab::ParallelComm::get_moab(), graph_type, INITIAL_MIGRATE, involved_IDs_map, localSendBuffs, map_index, map_ptr, mb, MB_CHK_ERR, MB_SET_ERR, MB_SUCCESS, rankInJoin, sendReqs, moab::Range::size(), and split_ranges.

Referenced by iMOAB_SendElementTag().

◆ sender()

int moab::ParCommGraph::sender ( int index )

inline

Definition at line 143 of file ParCommGraph.hpp.

     {
         return senderTasks[index];
     }

References senderTasks.

Referenced by receive_comm_graph().

◆ senders()

const std::vector< int >& moab::ParCommGraph::senders ( )

inline

Definition at line 206 of file ParCommGraph.hpp.

     {
         return senderTasks;
     }  // reference copy; refers to sender tasks in joint comm

References senderTasks.

Referenced by pack_receivers_graph(), and send_graph_partition().

◆ set_context_id()

void moab::ParCommGraph::set_context_id ( int other_id )

inline

Definition at line 166 of file ParCommGraph.hpp.

     {
         context_id = other_id;
     }

References context_id.

◆ set_cover_set()

void moab::ParCommGraph::set_cover_set ( EntityHandle cover )

inline

Definition at line 175 of file ParCommGraph.hpp.

     {
         cover_set = cover;
     }

References cover_set.

◆ SetReceivingAfterCoverage()

void moab::ParCommGraph::SetReceivingAfterCoverage ( std::map< int, std::set< int > > & idsFromProcs )

Definition at line 1023 of file ParCommGraph.cpp.

 {
     for( auto mt = idsFromProcs.begin(); mt != idsFromProcs.end(); ++mt )
     {
         int fromProc            = mt->first;
         std::set< int >& setIds = mt->second;
         involved_IDs_map[fromProc].resize( setIds.size() );
         std::vector< int >& listIDs = involved_IDs_map[fromProc];
         size_t indx                 = 0;
         for( std::set< int >::iterator st = setIds.begin(); st != setIds.end(); st++ )
         {
             int valueID     = *st;
             listIDs[indx++] = valueID;
         }
     }
     graph_type = COVERAGE;
     return;
 }

References COVERAGE, graph_type, and involved_IDs_map.

◆ settle_comm_by_ids()

void moab::ParCommGraph::settle_comm_by_ids	(	int	comp,
		TupleList &	TLBackToComp,
		std::vector< int > &	valuesComp
	)

Definition at line 1043 of file ParCommGraph.cpp.

 {
     // settle comm graph on comp
     if( rootSender || rootReceiver ) std::cout << " settle comm graph by id on component " << comp << "\n";
     int n = TLBackToComp.get_n();
     // third_method = true; // do not rely only on involved_IDs_map.size(); this can be 0 in some
     // cases
     std::map< int, std::set< int > > uniqueIDs;
     for( int i = 0; i < n; i++ )
     {
         int to_proc  = TLBackToComp.vi_wr[3 * i + 2];
         int globalId = TLBackToComp.vi_wr[3 * i + 1];
         uniqueIDs[to_proc].insert( globalId );
     }
  
     // Vector to store element
     // with respective present index
     std::vector< std::pair< int, int > > vp;
     vp.reserve( valuesComp.size() );
  
     // Inserting element in pair vector
     // to keep track of previous indexes in valuesComp
     for( size_t i = 0; i < valuesComp.size(); ++i )
     {
         vp.push_back( std::make_pair( valuesComp[i], i ) );
     }
     // Sorting pair vector
     sort( vp.begin(), vp.end() );
  
     // vp[i].first, second
  
     // count now how many times some value appears in ordered (so in valuesComp)
     for( auto it = uniqueIDs.begin(); it != uniqueIDs.end(); ++it )
     {
         int procId                  = it->first;
         std::set< int >& nums       = it->second;
         std::vector< int >& indx    = map_ptr[procId];
         std::vector< int >& indices = map_index[procId];
         indx.resize( nums.size() + 1 );
         int indexInVp = 0;
         int indexVal  = 0;
         indx[0]       = 0;  // start from 0
         for( auto sst = nums.begin(); sst != nums.end(); ++sst, ++indexVal )
         {
             int val = *sst;
             involved_IDs_map[procId].push_back( val );
             indx[indexVal + 1] = indx[indexVal];
             while( ( indexInVp < (int)valuesComp.size() ) && ( vp[indexInVp].first <= val ) )  // should be equal !
             {
                 if( vp[indexInVp].first == val )
                 {
                     indx[indexVal + 1]++;
                     indices.push_back( vp[indexInVp].second );
                 }
                 indexInVp++;
             }
         }
     }
 #ifdef VERBOSE
     std::stringstream f1;
     std::ofstream dbfile;
     f1 << "Involve_" << comp << "_" << rankInJoin << ".txt";
     dbfile.open( f1.str().c_str() );
     for( auto mit = involved_IDs_map.begin(); mit != involved_IDs_map.end(); ++mit )
     {
         int corrTask                = mit->first;
         std::vector< int >& corrIds = mit->second;
         std::vector< int >& indx    = map_ptr[corrTask];
         std::vector< int >& indices = map_index[corrTask];
  
         dbfile << " towards proc " << corrTask << " \n";
         for( int i = 0; i < (int)corrIds.size(); i++ )
         {
             dbfile << corrIds[i] << " [" << indx[i] << "," << indx[i + 1] << ")  : ";
             for( int j = indx[i]; j < indx[i + 1]; j++ )
                 dbfile << indices[j] << " ";
             dbfile << "\n";
         }
         dbfile << " \n";
     }
     dbfile.close();
 #endif
  
     graph_type = DOF_BASED;
     // now we need to fill back and forth information, needed to fill the arrays
     // for example, from spectral to involved_IDs_map, in case we want to send data from
     // spectral to phys
 }

References DOF_BASED, moab::GeomUtil::first(), moab::TupleList::get_n(), graph_type, involved_IDs_map, map_index, map_ptr, rankInJoin, rootReceiver, rootSender, and moab::TupleList::vi_wr.

Referenced by iMOAB_ComputeCommGraph().

◆ settle_send_graph()

ErrorCode moab::ParCommGraph::settle_send_graph ( TupleList & TLcovIDs )

Definition at line 995 of file ParCommGraph.cpp.

 {
     // fill involved_IDs_map with data
     // will have "receiving proc" and global id of element
     int n      = TLcovIDs.get_n();
     graph_type = COVERAGE;  // do not rely only on involved_IDs_map.size(); this can be 0 in some cases
     for( int i = 0; i < n; i++ )
     {
         int to_proc      = TLcovIDs.vi_wr[2 * i];
         int globalIdElem = TLcovIDs.vi_wr[2 * i + 1];
         involved_IDs_map[to_proc].push_back( globalIdElem );
     }
 #ifdef VERBOSE
     for( std::map< int, std::vector< int > >::iterator mit = involved_IDs_map.begin(); mit != involved_IDs_map.end();
          ++mit )
     {
         std::cout << " towards task " << mit->first << " send: " << mit->second.size() << " cells " << std::endl;
         for( size_t i = 0; i < mit->second.size(); i++ )
         {
             std::cout << " " << mit->second[i];
         }
         std::cout << std::endl;
     }
 #endif
     return MB_SUCCESS;
 }

References COVERAGE, moab::TupleList::get_n(), graph_type, involved_IDs_map, MB_SUCCESS, and moab::TupleList::vi_wr.

◆ split_owned_range() [1/2]

ErrorCode moab::ParCommGraph::split_owned_range	(	int	sender_rank,
		Range &	owned
	)

Definition at line 217 of file ParCommGraph.cpp.

 {
     int senderTask                   = senderTasks[sender_rank];
     std::vector< int >& distribution = sender_sizes[senderTask];
     std::vector< int >& receivers    = sender_graph[senderTask];
     if( distribution.size() != receivers.size() )  //
         return MB_FAILURE;
  
     Range current = owned;  // get the full range first, then we will subtract stuff, for
     // the following ranges
  
     Range rleftover = current;
     for( size_t k = 0; k < receivers.size(); k++ )
     {
         Range newr;
         newr.insert( current.begin(), current.begin() + distribution[k] );
         split_ranges[receivers[k]] = newr;
  
         rleftover = subtract( current, newr );
         current   = rleftover;
     }
  
     return MB_SUCCESS;
 }

References moab::Range::begin(), moab::Range::insert(), MB_SUCCESS, receivers(), sender_graph, sender_sizes, senderTasks, split_ranges, and moab::subtract().

Referenced by send_mesh_parts().

◆ split_owned_range() [2/2]

ErrorCode moab::ParCommGraph::split_owned_range ( Range & owned )

Definition at line 243 of file ParCommGraph.cpp.

 {
     if( corr_tasks.size() != corr_sizes.size() )  //
         return MB_FAILURE;
  
     Range current = owned;  // get the full range first, then we will subtract stuff, for
     // the following ranges
  
     Range rleftover = current;
     for( size_t k = 0; k < corr_tasks.size(); k++ )
     {
         Range newr;
         newr.insert( current.begin(), current.begin() + corr_sizes[k] );
         split_ranges[corr_tasks[k]] = newr;
  
         rleftover = subtract( current, newr );
         current   = rleftover;
     }
  
     return MB_SUCCESS;
 }

References moab::Range::begin(), corr_sizes, corr_tasks, moab::Range::insert(), MB_SUCCESS, split_ranges, and moab::subtract().

Member Data Documentation

◆ comm

MPI_Comm moab::ParCommGraph::comm

private

Definition at line 244 of file ParCommGraph.hpp.

Referenced by ParCommGraph().

◆ comm_graph

int* moab::ParCommGraph::comm_graph

private

Definition at line 269 of file ParCommGraph.hpp.

Referenced by ParCommGraph(), release_send_buffers(), and send_graph().

◆ compid1

int moab::ParCommGraph::compid1

private

Definition at line 251 of file ParCommGraph.hpp.

Referenced by get_component_id1(), ParCommGraph(), receive_tag_values(), send_graph_partition(), and send_tag_values().

◆ compid2

int moab::ParCommGraph::compid2

private

Definition at line 251 of file ParCommGraph.hpp.

Referenced by get_component_id2(), ParCommGraph(), receive_comm_graph(), receive_mesh(), receive_tag_values(), send_graph(), send_graph_partition(), send_mesh_parts(), and send_tag_values().

◆ context_id

int moab::ParCommGraph::context_id

private

Definition at line 252 of file ParCommGraph.hpp.

Referenced by get_context_id(), ParCommGraph(), and set_context_id().

◆ corr_sizes

std::vector< int > moab::ParCommGraph::corr_sizes

private

Definition at line 283 of file ParCommGraph.hpp.

Referenced by receive_mesh(), send_mesh_parts(), and split_owned_range().

◆ corr_tasks

std::vector< int > moab::ParCommGraph::corr_tasks

private

Definition at line 281 of file ParCommGraph.hpp.

Referenced by receive_mesh(), send_mesh_parts(), and split_owned_range().

◆ cover_set

EntityHandle moab::ParCommGraph::cover_set

private

Definition at line 253 of file ParCommGraph.hpp.

Referenced by get_cover_set(), ParCommGraph(), and set_cover_set().

◆ graph_type

TypeGraph moab::ParCommGraph::graph_type

private

Definition at line 289 of file ParCommGraph.hpp.

Referenced by dump_comm_information(), ParCommGraph(), receive_tag_values(), send_tag_values(), SetReceivingAfterCoverage(), settle_comm_by_ids(), and settle_send_graph().

◆ involved_IDs_map

std::map< int, std::vector< int > > moab::ParCommGraph::involved_IDs_map

private

Definition at line 290 of file ParCommGraph.hpp.

Referenced by dump_comm_information(), receive_tag_values(), send_tag_values(), SetReceivingAfterCoverage(), settle_comm_by_ids(), and settle_send_graph().

◆ joinSize

int moab::ParCommGraph::joinSize

private

Definition at line 250 of file ParCommGraph.hpp.

Referenced by ParCommGraph().

◆ localSendBuffs

std::vector< ParallelComm::Buffer* > moab::ParCommGraph::localSendBuffs

private

Definition at line 266 of file ParCommGraph.hpp.

Referenced by release_send_buffers(), send_mesh_parts(), and send_tag_values().

◆ map_index

std::map< int, std::vector< int > > moab::ParCommGraph::map_index

private

Definition at line 293 of file ParCommGraph.hpp.

Referenced by receive_tag_values(), send_tag_values(), and settle_comm_by_ids().

◆ map_ptr

std::map< int, std::vector< int > > moab::ParCommGraph::map_ptr

private

Definition at line 294 of file ParCommGraph.hpp.

Referenced by receive_tag_values(), send_tag_values(), and settle_comm_by_ids().

◆ rankInGroup1

int moab::ParCommGraph::rankInGroup1

private

Definition at line 249 of file ParCommGraph.hpp.

Referenced by dump_comm_information(), ParCommGraph(), and send_mesh_parts().

◆ rankInGroup2

int moab::ParCommGraph::rankInGroup2

private

Definition at line 249 of file ParCommGraph.hpp.

Referenced by dump_comm_information(), and ParCommGraph().

◆ rankInJoin

int moab::ParCommGraph::rankInJoin

private

Definition at line 250 of file ParCommGraph.hpp.

Referenced by dump_comm_information(), ParCommGraph(), receive_mesh(), receive_tag_values(), send_tag_values(), and settle_comm_by_ids().

◆ receiverTasks

std::vector< int > moab::ParCommGraph::receiverTasks

private

Definition at line 246 of file ParCommGraph.hpp.

Referenced by compute_partition(), compute_trivial_partition(), pack_receivers_graph(), ParCommGraph(), receiver(), and receivers().

◆ recv_graph

std::map< int, std::vector< int > > moab::ParCommGraph::recv_graph

private

Definition at line 258 of file ParCommGraph.hpp.

Referenced by compute_trivial_partition(), pack_receivers_graph(), and send_graph_partition().

◆ recv_sizes

std::map< int, std::vector< int > > moab::ParCommGraph::recv_sizes

private

Definition at line 260 of file ParCommGraph.hpp.

Referenced by compute_trivial_partition().

◆ rootReceiver

bool moab::ParCommGraph::rootReceiver

private

Definition at line 248 of file ParCommGraph.hpp.

Referenced by is_root_receiver(), ParCommGraph(), receive_comm_graph(), and settle_comm_by_ids().

◆ rootSender

bool moab::ParCommGraph::rootSender

private

Definition at line 247 of file ParCommGraph.hpp.

Referenced by compute_partition(), is_root_sender(), ParCommGraph(), and settle_comm_by_ids().

◆ sender_graph

std::map< int, std::vector< int > > moab::ParCommGraph::sender_graph

private

Definition at line 262 of file ParCommGraph.hpp.

Referenced by compute_trivial_partition(), send_mesh_parts(), and split_owned_range().

◆ sender_sizes

std::map< int, std::vector< int > > moab::ParCommGraph::sender_sizes

private

Definition at line 264 of file ParCommGraph.hpp.

Referenced by compute_trivial_partition(), send_mesh_parts(), and split_owned_range().

◆ senderTasks

std::vector< int > moab::ParCommGraph::senderTasks

private

Definition at line 245 of file ParCommGraph.hpp.

Referenced by compute_trivial_partition(), ParCommGraph(), send_graph_partition(), send_mesh_parts(), sender(), senders(), and split_owned_range().

◆ sendReqs

std::vector< MPI_Request > moab::ParCommGraph::sendReqs

private

Definition at line 277 of file ParCommGraph.hpp.

Referenced by receive_tag_values(), release_send_buffers(), send_graph(), send_mesh_parts(), and send_tag_values().

◆ split_ranges

std::map< int, Range > moab::ParCommGraph::split_ranges

private

Definition at line 275 of file ParCommGraph.hpp.

Referenced by compute_partition(), dump_comm_information(), receive_mesh(), receive_tag_values(), send_graph_partition(), send_mesh_parts(), send_tag_values(), and split_owned_range().

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Member Enumeration Documentation

◆ TypeGraph

Constructor & Destructor Documentation

◆ ~ParCommGraph()

◆ ParCommGraph() [1/2]

◆ ParCommGraph() [2/2]

Member Function Documentation

◆ compute_partition()

◆ compute_trivial_partition()

◆ dump_comm_information()

◆ find_group_ranks()

◆ get_component_id1()

◆ get_component_id2()

◆ get_context_id()

◆ get_cover_set()

◆ is_root_receiver()

◆ is_root_sender()

◆ pack_receivers_graph()

◆ receive_comm_graph()

◆ receive_mesh()

◆ receive_tag_values()

◆ receiver()

◆ receivers()

◆ release_send_buffers()

◆ send_graph()

◆ send_graph_partition()

◆ send_mesh_parts()

◆ send_tag_values()

◆ sender()

◆ senders()

◆ set_context_id()

◆ set_cover_set()

◆ SetReceivingAfterCoverage()

◆ settle_comm_by_ids()

◆ settle_send_graph()

◆ split_owned_range() [1/2]

◆ split_owned_range() [2/2]

Member Data Documentation

◆ comm

◆ comm_graph

◆ compid1

◆ compid2

◆ context_id

◆ corr_sizes

◆ corr_tasks

◆ cover_set

◆ graph_type

◆ involved_IDs_map

◆ joinSize

◆ localSendBuffs

◆ map_index

◆ map_ptr

◆ rankInGroup1

◆ rankInGroup2

◆ rankInJoin

◆ receiverTasks

◆ recv_graph

◆ recv_sizes

◆ rootReceiver

◆ rootSender

◆ sender_graph

◆ sender_sizes

◆ senderTasks

◆ sendReqs

◆ split_ranges