imoab_coupler_utils.hpp

Go to the documentation of this file.

/*
 * imoab_coupler_utils.hpp
 *
 * Created on: Aug. 22, 2020
 * \brief will contain utility methods for refactoring imoab*coupler tests, to avoid repetitive tasks
 * \ even migrate tests can use some of these utilities
 * 1) create_comm_group(int start, int end, int tag, MPI_Group& group, MPI_Comm& comm)
 *
 */
 
#ifndef TEST_PARALLEL_IMOAB_COUPLER_UTILS_HPP_
#define TEST_PARALLEL_IMOAB_COUPLER_UTILS_HPP_
 
#define CHECKIERR( rc, message ) \
 if( 0 != ( rc ) ) \
 { \
 printf( "%s. ErrorCode = %d\n", message, rc ); \
 return 1; \
 }
 
#define PUSH_TIMER( operation ) \
 { \
 timer_ops = timer.time_since_birth(); \
 opName = operation; \
 }
#define POP_TIMER( localcomm, localrank ) \
 { \
 double locElapsed = timer.time_since_birth() - timer_ops, minElapsed = 0, maxElapsed = 0; \
 MPI_Reduce( &locElapsed, &maxElapsed, 1, MPI_DOUBLE, MPI_MAX, 0, localcomm ); \
 MPI_Reduce( &locElapsed, &minElapsed, 1, MPI_DOUBLE, MPI_MIN, 0, localcomm ); \
 if( !( localrank ) ) \
 std::cout << "[LOG] Time taken to " << opName.c_str() << ": max = " << maxElapsed \
 << ", avg = " << ( maxElapsed + minElapsed ) / 2 << "\n"; \
 opName.clear(); \
 }
 
/*
 * \brief create an MPI group and an MPI communicator for the group, in the global communicator
 */
int create_group_and_comm( int start, int end, MPI_Group worldGroup, MPI_Group* group, MPI_Comm* comm )
{
 std::vector< int > groupTasks;
 groupTasks.resize( end - start + 1, 0 );
 for( int i = start; i <= end; i++ )
 groupTasks[i - start] = i;
 
 int ierr = MPI_Group_incl( worldGroup, end - start + 1, &groupTasks[0], group );
 CHECKIERR( ierr, "Cannot create group" )
 
 ierr = MPI_Comm_create( MPI_COMM_WORLD, *group, comm );
 CHECKIERR( ierr, "Cannot create comm" )
 
 return 0;
}
 
int create_joint_comm_group( MPI_Group agroup, MPI_Group bgroup, MPI_Group* abgroup, MPI_Comm* abcomm )
{
 int ierr = MPI_Group_union( agroup, bgroup, abgroup );
 CHECKIERR( ierr, "Cannot create joint union group" )
 
 ierr = MPI_Comm_create( MPI_COMM_WORLD, *abgroup, abcomm );
 CHECKIERR( ierr, "Cannot create joint communicator from union group" )
 
 return 0;
}
 
int setup_component_coupler_meshes( iMOAB_AppID cmpId,
 int cmpTag,
 iMOAB_AppID cplCmpId,
 int cmpcouTag,
 MPI_Comm* cmpcomm,
 MPI_Group* cmpPEGroup,
 MPI_Comm* coucomm,
 MPI_Group* cplPEGroup,
 MPI_Comm* cmpcoucomm,
 std::string& filename,
 std::string& readopts,
 int nghlay,
 int repartitioner_scheme )
{
 int ierr = 0;
 if( *cmpcomm != MPI_COMM_NULL )
 {
 // load first mesh
 ierr = iMOAB_LoadMesh( cmpId, filename.c_str(), readopts.c_str(), &nghlay );
 CHECKIERR( ierr, "Cannot load component mesh" )
 
 // then send mesh to coupler pes
 ierr = iMOAB_SendMesh( cmpId, cmpcoucomm, cplPEGroup, &cmpcouTag,
 &repartitioner_scheme ); // send to coupler pes
 CHECKIERR( ierr, "cannot send elements" )
 }
 // now, receive mesh, on coupler communicator; first mesh 1, atm
 if( *coucomm != MPI_COMM_NULL )
 {
 
 ierr = iMOAB_ReceiveMesh( cplCmpId, cmpcoucomm, cmpPEGroup,
 &cmpTag ); // receive from component
 CHECKIERR( ierr, "cannot receive elements on coupler app" )
 }
 
 // we can now free the sender buffers
 if( *cmpcomm != MPI_COMM_NULL )
 {
 int context_id = cmpcouTag;
 ierr = iMOAB_FreeSenderBuffers( cmpId, &context_id );
 CHECKIERR( ierr, "cannot free buffers used to send atm mesh" )
 }
 return 0;
}
 
#endif /* TEST_PARALLEL_IMOAB_COUPLER_UTILS_HPP_ */