parallel_unit_tests.cpp

Go to the documentation of this file.

#include "moab/ParallelComm.hpp"
#include "MBParallelConventions.h"
#include "moab/ParCommGraph.hpp"
#include "ReadParallel.hpp"
#include "moab/FileOptions.hpp"
#include "MBTagConventions.hpp"
#include "moab/Core.hpp"
#include "moab_mpi.h"
#include "TestUtil.hpp"
 
#include <iostream>
#include <algorithm>
#include <map>
#include <sstream>
#include <cassert>
#if !defined( _MSC_VER ) && !defined( __MINGW32__ )
#include <unistd.h>
#endif
 
using namespace moab;
 
#define CHKERR( a ) \
 do \
 { \
 ErrorCode val = ( a ); \
 if( MB_SUCCESS != val ) \
 { \
 std::cerr << "Error code " << val << " at " << __FILE__ << ":" << __LINE__ << std::endl; \
 return val; \
 } \
 } while( false )
 
#define PCHECK( A ) \
 if( is_any_proc_error( !( A ) ) ) return report_error( __FILE__, __LINE__ )
 
ErrorCode report_error( const char* file, int line )
{
 std::cerr << "Failure at " << file << ':' << line << std::endl;
 return MB_FAILURE;
}
 
/**************************************************************************
 Utility Method Declarations
 **************************************************************************/
 
// Get processors an entity is shared with.
ErrorCode get_sharing_processors( Interface& moab, EntityHandle entity, std::vector< int >& other_procs_out );
 
// Create a parallel mesh
//
// Each processor will create four quads.
// Groups of four quads will be arranged as follows:
// +------+------+------+------+------+-----
// | | |
// | | |
// + Proc 0 + Proc 2 + Proc 4
// | | |
// | | |
// +------+------+------+------+------+-----
// | | |
// | | |
// + Proc 1 + Proc 3 + Proc 5
// | | |
// | | |
// +------+------+------+------+------+-----
//
// Vertices will be enumerated as follows:
// 1------6-----11-----16-----21-----26-----
// | | |
// | | |
// 2 7 12 17 22 27
// | | |
// | | |
// 3------8-----13-----18-----23-----28-----
// | | |
// | | |
// 4 9 14 19 24 29
// | | |
// | | |
// 5-----10-----15-----20-----25-----30-----
//
// Element IDs will be [4*rank+1,4*rank+5]
//
// If output_sets is non-null, it will be populated with 2 or 3
// set handles. A set handle is created for each group of four
// adjacent processes, such that one is created across procs 0 to 4,
// one is created for procs 2 to 5, etc. An additional set is created
// that spans all of the processes. The set spanning all procs is
// returned first. The other two sets are returned in the order of the
// largest contained process rank, where the last entry is zero if
// there is only one additional set.
ErrorCode parallel_create_mesh( Interface& mb,
 int output_vertx_ids[9],
 EntityHandle output_vertex_handles[9],
 Range& output_elements,
 EntityHandle output_sets[3] = 0 );
 
// Test if is_my_error is non-zero on any processor in MPI_COMM_WORLD
int is_any_proc_error( int is_my_error );
 
/**************************************************************************
 Test Declarations
 **************************************************************************/
 
// Check consistancy of sharing data. (E.g. compare sharing procs for
// vertices to that of adjacent elements, compare sharing data for
// interfaces with that of contained entities, etc.)
ErrorCode test_elements_on_several_procs( const char* filename );
// Test correct ghosting of elements
ErrorCode test_ghost_elements_3_2_1( const char* filename );
ErrorCode test_ghost_elements_3_2_2( const char* filename );
ErrorCode test_ghost_elements_3_0_1( const char* filename );
ErrorCode test_ghost_elements_2_0_1( const char* filename );
// Test exchange of tag data on ghost elements
ErrorCode test_ghost_tag_exchange( const char* filename );
// Bug where exchange_tags fails if dense tag cannot be queried
// for all ghost entities (e.g. no default value)
ErrorCode regression_ghost_tag_exchange_no_default( const char* filename );
// Test owners for interface entities
ErrorCode test_interface_owners( const char* );
// Test data for shared interface entitites with one level of ghosting
ErrorCode regression_owners_with_ghosting( const char* );
// Verify all sharing data for vertices with one level of ghosting
ErrorCode test_ghosted_entity_shared_data( const char* );
// Test assignment of global IDs
ErrorCode test_assign_global_ids( const char* );
// Test shared sets
ErrorCode test_shared_sets( const char* );
// Test reduce_tags
ErrorCode test_reduce_tags( const char* );
// Test reduce_tags failures
ErrorCode test_reduce_tag_failures( const char* );
// Test reduce_tags with explicit destination tag
ErrorCode test_reduce_tag_explicit_dest( const char* );
// Test delete_entities
ErrorCode test_delete_entities( const char* );
// Test ghosting polyhedra
ErrorCode test_ghost_polyhedra( const char* );
// Test failed read with too few parts in partition
ErrorCode test_too_few_parts( const char* );
// Test broken sequences due to ghosting
ErrorCode test_sequences_after_ghosting( const char* );
// Test trivial partition in use by iMOAB
void test_trivial_partition();
 
/**************************************************************************
 Main Method
 **************************************************************************/
 
#define RUN_TEST_ARG2( A, B ) run_test( &( A ), #A, B )
 
int run_test( ErrorCode ( *func )( const char* ), const char* func_name, const char* file_name )
{
 ErrorCode result = ( *func )( file_name );
 int is_err = is_any_proc_error( ( MB_SUCCESS != result ) );
 int rank;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 if( rank == 0 )
 {
 if( is_err )
 std::cout << func_name << " : FAILED!!" << std::endl;
 else
 std::cout << func_name << " : success" << std::endl;
 }
 
 return is_err;
}
 
int main( int argc, char* argv[] )
{
 int rank, size;
 MPI_Init( &argc, &argv );
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 int pause_proc = -1;
 std::string filename;
 for( int i = 1; i < argc; ++i )
 {
 if( !strcmp( argv[i], "-p" ) )
 {
 ++i;
 assert( i < argc );
 pause_proc = atoi( argv[i] );
 }
 else if( !filename.size() )
 {
 filename = std::string( argv[i] );
 }
 else
 {
 std::cerr << "Invalid arg: \"" << argv[i] << '"' << std::endl
 << "Usage: " << argv[0] << " [-p <rank>] [<filename>]" << std::endl;
 exit( 1 );
 }
 }
 
 if( !filename.size() )
 {
#ifdef MOAB_HAVE_HDF5
 filename = TestDir + "unittest/64bricks_512hex.h5m";
#else
 filename = TestDir + "unittest/64bricks_512hex.vtk";
#endif
 }
 std::cout << "Loading " << filename << "..\n";
#ifdef MOAB_HAVE_HDF5
 std::string filename2 = TestDir + "unittest/64bricks_1khex.h5m";
 std::string filename3 = TestDir + "unittest/twoPolyh.h5m";
 std::string filename4 = TestDir + "unittest/onepart.h5m";
#endif
 
 if( pause_proc != -1 )
 {
#if !defined( _MSC_VER ) && !defined( __MINGW32__ )
 std::cout << "Processor " << rank << " of " << size << " with PID " << getpid() << std::endl;
 std::cout.flush();
#endif
 // loop forever on requested processor, giving the user time
 // to attach a debugger. Once the debugger in attached, user
 // can change 'pause'. E.g. on gdb do "set var pause = 0"
 if( pause_proc == rank )
 {
 volatile int pause = 1;
 while( pause )
 ;
 }
 
 MPI_Barrier( MPI_COMM_WORLD );
 std::cout << "Processor " << rank << " resuming" << std::endl;
 }
 
 int num_errors = 0;
#ifdef MOAB_HAVE_HDF5
 num_errors += RUN_TEST_ARG2( test_elements_on_several_procs, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_ghost_elements_3_2_1, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_ghost_elements_3_2_2, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_ghost_elements_3_0_1, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_ghost_elements_2_0_1, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_ghost_tag_exchange, filename.c_str() );
 num_errors += RUN_TEST_ARG2( regression_ghost_tag_exchange_no_default, filename.c_str() );
 num_errors += RUN_TEST_ARG2( test_delete_entities, filename2.c_str() );
 num_errors += RUN_TEST_ARG2( test_sequences_after_ghosting, filename2.c_str() );
 if( 2 >= size ) // run this one only on one or 2 processors; the file has only 2 parts in partition
 num_errors += RUN_TEST_ARG2( test_ghost_polyhedra, filename3.c_str() );
 num_errors += RUN_TEST_ARG2( test_too_few_parts, filename4.c_str() );
#endif
 num_errors += RUN_TEST_ARG2( test_assign_global_ids, 0 );
 num_errors += RUN_TEST_ARG2( test_shared_sets, 0 );
 num_errors += RUN_TEST_ARG2( test_reduce_tags, 0 );
 num_errors += RUN_TEST_ARG2( test_reduce_tag_failures, 0 );
 num_errors += RUN_TEST_ARG2( test_reduce_tag_explicit_dest, 0 );
 num_errors += RUN_TEST_ARG2( test_interface_owners, 0 );
 num_errors += RUN_TEST_ARG2( test_ghosted_entity_shared_data, 0 );
 num_errors += RUN_TEST_ARG2( regression_owners_with_ghosting, 0 );
 num_errors += RUN_TEST( test_trivial_partition );
 if( rank == 0 )
 {
 if( !num_errors )
 std::cout << "All tests passed" << std::endl;
 else
 std::cout << num_errors << " TESTS FAILED!" << std::endl;
 }
 
 MPI_Finalize();
 return num_errors;
}
 
/**************************************************************************
 Utility Method Implementations
 **************************************************************************/
 
ErrorCode get_sharing_processors( Interface& moab, EntityHandle entity, std::vector< int >& other_procs_out )
{
 ErrorCode rval;
 
 // get tags for parallel data
 Tag sharedp_tag, sharedps_tag, sharedh_tag, sharedhs_tag, pstatus_tag;
 rval = moab.tag_get_handle( PARALLEL_SHARED_PROC_TAG_NAME, 1, MB_TYPE_INTEGER, sharedp_tag );CHKERR( rval );
 rval = moab.tag_get_handle( PARALLEL_SHARED_PROCS_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_INTEGER, sharedps_tag );CHKERR( rval );
 rval = moab.tag_get_handle( PARALLEL_SHARED_HANDLE_TAG_NAME, 1, MB_TYPE_HANDLE, sharedh_tag );CHKERR( rval );
 rval = moab.tag_get_handle( PARALLEL_SHARED_HANDLES_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_HANDLE, sharedhs_tag );CHKERR( rval );
 rval = moab.tag_get_handle( PARALLEL_STATUS_TAG_NAME, 1, MB_TYPE_OPAQUE, pstatus_tag );CHKERR( rval );
 
 other_procs_out.clear();
 char status;
 rval = moab.tag_get_data( pstatus_tag, &entity, 1, &status );CHKERR( rval );
 if( !( status & PSTATUS_SHARED ) ) return MB_SUCCESS;
 
 int proc_id;
 rval = moab.tag_get_data( sharedp_tag, &entity, 1, &proc_id );CHKERR( rval );
 if( proc_id >= 0 )
 {
 other_procs_out.push_back( proc_id );
 return MB_SUCCESS;
 }
 
 int procs[MAX_SHARING_PROCS];
 rval = moab.tag_get_data( sharedps_tag, &entity, 1, procs );CHKERR( rval );
 for( int i = 0; i < MAX_SHARING_PROCS && procs[i] >= 0; ++i )
 other_procs_out.push_back( procs[i] );
 return MB_SUCCESS;
}
 
int is_any_proc_error( int is_my_error )
{
 int result = 0;
 int err = MPI_Allreduce( &is_my_error, &result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD );
 return err || result;
}
 
ErrorCode parallel_create_mesh( Interface& mb,
 int vtx_ids[9],
 EntityHandle vtx_handles[9],
 Range& range,
 EntityHandle* entity_sets )
{
 // Each processor will create four quads.
 // Groups of four quads will be arranged as follows:
 // +------+------+------+------+------+-----
 // | | |
 // | | |
 // + Proc 0 + Proc 2 + Proc 4
 // | | |
 // | | |
 // +------+------+------+------+------+-----
 // | | |
 // | | |
 // + Proc 1 + Proc 3 + Proc 5
 // | | |
 // | | |
 // +------+------+------+------+------+-----
 //
 // Vertices will be enumerated as follows:
 // 1------6-----11-----16-----21-----26-----
 // | | |
 // | | |
 // 2 7 12 17 22 27
 // | | |
 // | | |
 // 3------8-----13-----18-----23-----28-----
 // | | |
 // | | |
 // 4 9 14 19 24 29
 // | | |
 // | | |
 // 5-----10-----15-----20-----25-----30-----
 //
 // Element IDs will be [4*rank+1,4*rank+5]
 
 int size, rank;
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 
 const int first_vtx_id = 10 * ( rank / 2 ) + 2 * ( rank % 2 ) + 1;
 const double x = 2.0 * ( rank / 2 );
 const double y = 2.0 * ( rank % 2 );
 
 // create vertices
 const int idoff = ( size % 2 && rank / 2 == size / 2 ) ? 0 : 2;
 const int idoff1 = rank ? 2 : idoff;
 const int idoff2 = idoff1 + idoff;
 const int ids[9] = { first_vtx_id, first_vtx_id + 3 + idoff1, first_vtx_id + 6 + idoff2,
 first_vtx_id + 1, first_vtx_id + 4 + idoff1, first_vtx_id + 7 + idoff2,
 first_vtx_id + 2, first_vtx_id + 5 + idoff1, first_vtx_id + 8 + idoff2 };
 memcpy( vtx_ids, ids, sizeof( ids ) );
 const double coords[27] = { x, y, 0, x + 1, y, 0, x + 2, y, 0, x, y + 1, 0, x + 1, y + 1,
 0, x + 2, y + 1, 0, x, y + 2, 0, x + 1, y + 2, 0, x + 2, y + 2, 0 };
 
 ErrorCode rval;
 Tag id_tag;
 
 rval = mb.create_vertices( coords, 9, range );CHKERR( rval );
 assert( range.size() == 9 );
 std::copy( range.begin(), range.end(), vtx_handles );
 range.clear();
 id_tag = mb.globalId_tag();
 rval = mb.tag_set_data( id_tag, vtx_handles, 9, &ids );CHKERR( rval );
 
 const EntityHandle conn[4][4] = { { vtx_handles[0], vtx_handles[3], vtx_handles[4], vtx_handles[1] },
 { vtx_handles[1], vtx_handles[4], vtx_handles[5], vtx_handles[2] },
 { vtx_handles[3], vtx_handles[6], vtx_handles[7], vtx_handles[4] },
 { vtx_handles[4], vtx_handles[7], vtx_handles[8], vtx_handles[5] } };
 for( int i = 0; i < 4; ++i )
 {
 const int id = 4 * rank + i + 1;
 EntityHandle h;
 rval = mb.create_element( MBQUAD, conn[i], 4, h );CHKERR( rval );
 range.insert( h );
 rval = mb.tag_set_data( id_tag, &h, 1, &id );CHKERR( rval );
 }
 
 if( !entity_sets ) return MB_SUCCESS;
 
 int set_ids[3] = { size + 1, rank / 2, rank / 2 + 1 };
 int nsets = 0;
 rval = mb.create_meshset( MESHSET_SET, entity_sets[nsets++] );CHKERR( rval );
 rval = mb.create_meshset( MESHSET_SET, entity_sets[nsets++] );CHKERR( rval );
 entity_sets[nsets] = 0;
 if( rank < 2 )
 { // if first (left-most) column
 set_ids[1] = set_ids[2];
 }
 else if( rank / 2 < ( size - 1 ) / 2 )
 { // if not last (right-most) column
 rval = mb.create_meshset( MESHSET_SET, entity_sets[nsets++] );CHKERR( rval );
 }
 for( int i = 0; i < nsets; ++i )
 {
 rval = mb.add_entities( entity_sets[0], range );CHKERR( rval );
 }
 rval = mb.tag_set_data( id_tag, entity_sets, nsets, set_ids );CHKERR( rval );
 
 return MB_SUCCESS;
}
 
/**************************************************************************
 Test Implementations
 **************************************************************************/
 
ErrorCode test_elements_on_several_procs( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 const char* geom_names[] = { "vertex", "curve", "surface", "volume", "unknown" };
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_DELETE;"
 "PARTITION=GEOM_DIMENSION;PARTITION_VAL=3;"
 "PARTITION_DISTRIBUTE;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_SEQUENCE_FACTOR=1.4" );CHKERR( rval );
 
 // test contents of interface sets against sharedEnts structure in pcomm;
 int my_error = 0;
 ParallelComm* pcomm = ParallelComm::get_pcomm( &moab, 0 );
 rval = pcomm->check_all_shared_handles();
 if( MB_SUCCESS != rval )
 {
 my_error = 1;
 std::cerr << "check_all_shared_handles test failed on proc " << pcomm->proc_config().proc_rank() << std::endl;
 }
 PCHECK( !my_error );
 
 // check adjacencies just to make sure they're consistent
 rval = mb_instance.check_adjacencies();
 if( MB_SUCCESS != rval ) my_error = 1;
 PCHECK( !my_error );
 
 Tag geom_tag, id_tag;
 rval = moab.tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, geom_tag );CHKERR( rval );
 id_tag = moab.globalId_tag();
 
 // Because we partitioned based on geometric volume sets, then for each
 // lower-dimension geometric entity set all of the contained entities of
 // the same dimension must be shared by the same set of processors
 Range shared, invalid;
 for( int dim = 2; dim > 0; --dim )
 {
 Range geom_sets;
 const void* tagvals[] = { &dim };
 rval = moab.get_entities_by_type_and_tag( 0, MBENTITYSET, &geom_tag, tagvals, 1, geom_sets );CHKERR( rval );
 
 for( Range::iterator j = geom_sets.begin(); j != geom_sets.end(); ++j )
 {
 Range ents;
 rval = moab.get_entities_by_dimension( *j, dim, ents );CHKERR( rval );
 if( ents.empty() ) continue;
 
 // get a single sharing list to compare with
 Range::iterator k = ents.begin();
 std::vector< int > procs;
 rval = get_sharing_processors( moab, *k, procs );CHKERR( rval );
 std::sort( procs.begin(), procs.end() );
 if( procs.size() > 1 ) shared.merge( ents );
 
 // compare other elements
 for( ++k; k != ents.end(); ++k )
 {
 std::vector< int > tmp_procs;
 rval = get_sharing_processors( moab, *k, tmp_procs );CHKERR( rval );
 std::sort( tmp_procs.begin(), tmp_procs.end() );
 if( tmp_procs != procs ) invalid.insert( *j );
 }
 
 // compare interior vertices
 ents.clear();
 rval = moab.get_entities_by_dimension( *j, 0, ents );CHKERR( rval );
 for( k = ents.begin(); k != ents.end(); ++k )
 {
 std::vector< int > tmp_procs;
 rval = get_sharing_processors( moab, *k, tmp_procs );CHKERR( rval );
 if( tmp_procs != procs ) invalid.insert( *j );
 }
 }
 }
 
 // Report any geometric sets for which sharing was inconsistent
 if( !invalid.empty() )
 {
 std::cerr << "Elements or vertices owned by a single geometric entity are "
 << "not shared by the same set of processors for the "
 << "following geometric entities on process " << pcomm->proc_config().proc_rank() << ": ";
 for( Range::iterator i = invalid.begin(); i != invalid.end(); ++i )
 {
 int dim;
 int id;
 rval = moab.tag_get_data( geom_tag, &*i, 1, &dim );
 if( MB_SUCCESS != rval ) dim = 4;
 rval = moab.tag_get_data( id_tag, &*i, 1, &id );
 if( MB_SUCCESS != rval ) id = -1;
 std::cerr << geom_names[dim] << " " << id << ", ";
 }
 std::cerr << std::endl;
 
 my_error = 1;
 }
 PCHECK( !my_error );
 
 // for each shared element, check that its vertices are shared
 // with at least the same processes
 for( Range::iterator i = shared.begin(); i != shared.end(); ++i )
 {
 std::vector< int > procs;
 rval = get_sharing_processors( moab, *i, procs );CHKERR( rval );
 std::sort( procs.begin(), procs.end() );
 
 std::vector< EntityHandle > tmp;
 const EntityHandle* conn;
 int len;
 rval = moab.get_connectivity( *i, conn, len, false, &tmp );CHKERR( rval );
 for( int j = 0; j < len; ++j )
 {
 std::vector< int > vprocs;
 rval = get_sharing_processors( moab, conn[j], vprocs );CHKERR( rval );
 std::sort( vprocs.begin(), vprocs.end() );
 std::vector< int > diff( std::max( procs.size(), vprocs.size() ) );
 std::vector< int >::iterator k =
 std::set_difference( procs.begin(), procs.end(), vprocs.begin(), vprocs.end(), diff.begin() );
 if( k != diff.begin() ) // difference is not empty
 invalid.insert( conn[j] );
 }
 }
 
 // Report any vertices with insufficient sharing
 if( !invalid.empty() )
 {
 std::cerr << "Vertices must be shared with at least the union of the processes "
 << "sharing the elements containing the vertex. This is NOT true for "
 << "the following vertices on process " << pcomm->proc_config().proc_rank() << ": " << invalid
 << std::endl;
 
 my_error = 1;
 }
 PCHECK( !my_error );
 
 return MB_SUCCESS;
}
 
ErrorCode get_ghost_entities( ParallelComm& pcomm, Range& ghost_ents )
{
 Range all_ents;
 ErrorCode rval;
 
 rval = pcomm.get_moab()->get_entities_by_handle( 0, all_ents );CHKERR( rval );
 std::vector< unsigned char > flags( all_ents.size() );
 rval = pcomm.get_moab()->tag_get_data( pcomm.pstatus_tag(), all_ents, &flags[0] );CHKERR( rval );
 
 Range::iterator ins = ghost_ents.begin();
 std::vector< unsigned char >::const_iterator f = flags.begin();
 for( Range::iterator i = all_ents.begin(); i != all_ents.end(); ++i, ++f )
 if( ( *f & PSTATUS_NOT_OWNED ) && !( *f & PSTATUS_INTERFACE ) ) ins = ghost_ents.insert( ins, *i, *i );
 
 return MB_SUCCESS;
}
 
ErrorCode get_ents_from_geometric_sets( Interface& moab,
 const Tag tags[2],
 int dimension,
 const std::vector< int >& ids,
 Range& results )
{
 ErrorCode rval;
 for( size_t i = 0; i < ids.size(); ++i )
 {
 const void* tag_vals[2] = { &dimension, &ids[i] };
 Range sets;
 rval = moab.get_entities_by_type_and_tag( 0, MBENTITYSET, tags, tag_vals, 2, sets );CHKERR( rval );
 for( Range::iterator j = sets.begin(); j != sets.end(); ++j )
 {
 Range ents;
 rval = moab.get_entities_by_dimension( *j, dimension, ents );CHKERR( rval );
 results.merge( ents );
 }
 }
 return MB_SUCCESS;
}
 
/**\brief Given entire file and IDs of geometric sets, get expected ghost entities
 *
 *\param moab The entire mesh, loaded in serial
 *\param partition_geom_ids IDs of: geometric volumes owned by this proc and interface topology
 *\param ghost_entity_ids output list
 */
ErrorCode get_expected_ghosts( Interface& moab,
 const std::vector< int > partition_geom_ids[4],
 std::vector< int >& ghost_entity_ids,
 int ghost_dimension,
 int bridge_dimension,
 int num_layers )
{
 ErrorCode rval;
 Tag tags[2];
 rval = moab.tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tags[0] );CHKERR( rval );
 tags[1] = moab.globalId_tag();
 
 // get face entities defining interface
 Range skin;
 rval = get_ents_from_geometric_sets( moab, tags, 2, partition_geom_ids[2], skin );CHKERR( rval );
 
 // get adjacent entities
 Range iface_ghosts, iface_ents;
 if( bridge_dimension == 2 )
 {
 iface_ents = skin;
 }
 else
 {
 rval = moab.get_adjacencies( skin, bridge_dimension, true, iface_ents, Interface::UNION );CHKERR( rval );
 }
 for( int n = 0; n < num_layers; ++n )
 {
 iface_ghosts.clear();
 rval = moab.get_adjacencies( iface_ents, ghost_dimension, false, iface_ghosts, Interface::UNION );CHKERR( rval );
 iface_ents.clear();
 rval = moab.get_adjacencies( iface_ghosts, bridge_dimension, true, iface_ents, Interface::UNION );CHKERR( rval );
 }
 
 // get volume entities owned by this process
 Range ents;
 rval = get_ents_from_geometric_sets( moab, tags, 3, partition_geom_ids[3], ents );CHKERR( rval );
 
 // get entities of ghost_dimension owned by this process
 Range owned;
 if( ghost_dimension == 3 )
 owned = ents;
 else
 {
 rval = moab.get_adjacencies( ents, ghost_dimension, false, owned, Interface::UNION );CHKERR( rval );
 }
 
 // remove owned entities from ghost list
 Range ghosts = subtract( iface_ghosts, owned );
 
 // get ids
 ghost_entity_ids.resize( ghosts.size() );
 rval = moab.tag_get_data( tags[1], ghosts, &ghost_entity_ids[0] );CHKERR( rval );
 return MB_SUCCESS;
}
 
ErrorCode test_ghost_elements( const char* filename, int ghost_dimension, int bridge_dimension, int num_layers )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 std::ostringstream file_opts;
 file_opts << "PARALLEL=READ_DELETE;"
 << "PARTITION=GEOM_DIMENSION;PARTITION_VAL=3;"
 << "PARTITION_DISTRIBUTE;"
 << "PARALLEL_RESOLVE_SHARED_ENTS;"
 << "PARALLEL_GHOSTS=" << ghost_dimension << '.' << bridge_dimension << '.' << num_layers;
 
 rval = moab.load_file( filename, 0, file_opts.str().c_str() );CHKERR( rval );
 Tag geom_tag, id_tag;
 rval = moab.tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, geom_tag );CHKERR( rval );
 id_tag = moab.globalId_tag();
 
 // Get partition sets
 Range partition_geom[4];
 ParallelComm* pcomm = ParallelComm::get_pcomm( &moab, 0 );
 partition_geom[3] = pcomm->partition_sets();
 PCHECK( !partition_geom[3].empty() );
 
 rval = pcomm->check_all_shared_handles();CHKERR( rval );
 
 // exchange id tags to allow comparison by id
 Range tmp_ents;
 rval = pcomm->get_shared_entities( -1, tmp_ents, -1, false, true );
 rval = pcomm->exchange_tags( id_tag, tmp_ents );CHKERR( rval );
 
 // Get geometric surfaces
 Range surfs, tmp;
 for( Range::iterator i = partition_geom[3].begin(); i != partition_geom[3].end(); ++i )
 {
 tmp.clear();
 rval = moab.get_child_meshsets( *i, tmp );CHKERR( rval );
 surfs.merge( tmp );
 }
 
 // Get the set of geometric surfaces that represent the skin
 // of the union of the parts for this processor. As we partition
 // based on geometric volumes, the interface must be represented
 // by some subset of the surfaces and their child geometric topology.
 
 int error = 0;
 std::ostringstream error_msg;
 Range ents, iface_surfs, iface_curves, iface_vertices;
 for( Range::iterator i = surfs.begin(); !error && i != surfs.end(); ++i )
 {
 ents.clear();
 rval = moab.get_entities_by_dimension( *i, ghost_dimension - 1, ents );CHKERR( rval );
 if( ents.empty() ) continue;
 
 std::vector< int > procs, tmp_procs;
 Range::iterator j = ents.begin();
 rval = get_sharing_processors( moab, *j, procs );CHKERR( rval );
 for( ++j; !error && j != ents.end(); ++j )
 {
 tmp_procs.clear();
 rval = get_sharing_processors( moab, *j, tmp_procs );CHKERR( rval );
 if( tmp_procs != procs )
 {
 error_msg << "Failure at " << __FILE__ << ':' << __LINE__ << std::endl
 << "\tNot all entities in geometric surface are shared with"
 << " same processor." << std::endl;
 error = 1;
 break;
 }
 }
 
 if( error ) break;
 
 // if surface is not part of inter-proc interface, skip it.
 if( procs.empty() ) continue;
 if( procs.size() != 1 )
 {
 error_msg << "Failure at " << __FILE__ << ':' << __LINE__ << std::endl
 << "\tSurface elements shared with" << procs.size() << "processors." << std::endl;
 error = 1;
 break;
 }
 int other_rank = procs[0];
 if( other_rank == (int)pcomm->proc_config().proc_rank() ) continue;
 
 partition_geom[2].insert( *i );
 ents.clear();
 rval = moab.get_child_meshsets( *i, ents );CHKERR( rval );
 partition_geom[1].merge( ents );
 }
 
 // Don't to global communication until outside
 // of loops. Otherwise we will deadlock if not all
 // procs iterate the same number of times.
 if( is_any_proc_error( error ) )
 {
 std::cerr << error_msg.str();
 return MB_FAILURE;
 }
 
 for( Range::iterator i = partition_geom[1].begin(); i != partition_geom[1].end(); ++i )
 {
 ents.clear();
 rval = moab.get_child_meshsets( *i, ents );CHKERR( rval );
 partition_geom[0].merge( ents );
 }
 
 std::vector< int > partn_geom_ids[4];
 for( int dim = 0; dim <= 3; ++dim )
 {
 partn_geom_ids[dim].resize( partition_geom[dim].size() );
 rval = moab.tag_get_data( id_tag, partition_geom[dim], &( partn_geom_ids[dim][0] ) );CHKERR( rval );
 }
 
 // get the global IDs of the ghosted entities
 Range ghost_ents;
 rval = get_ghost_entities( *pcomm, ghost_ents );CHKERR( rval );
 std::pair< Range::iterator, Range::iterator > vtx = ghost_ents.equal_range( MBVERTEX );
 ghost_ents.erase( vtx.first, vtx.second );
 std::vector< int > actual_ghost_ent_ids( ghost_ents.size() );
 rval = moab.tag_get_data( id_tag, ghost_ents, &actual_ghost_ent_ids[0] );CHKERR( rval );
 
 // read file in serial
 Core moab2;
 rval = moab2.load_file( filename );
 PCHECK( MB_SUCCESS == rval );
 
 // get the global IDs of the entities we expect to be ghosted
 std::vector< int > expected_ghost_ent_ids;
 rval = get_expected_ghosts( moab2, partn_geom_ids, expected_ghost_ent_ids, ghost_dimension, bridge_dimension,
 num_layers );
 PCHECK( MB_SUCCESS == rval );
 
 // check that the correct entities were ghosted
 std::sort( actual_ghost_ent_ids.begin(), actual_ghost_ent_ids.end() );
 std::sort( expected_ghost_ent_ids.begin(), expected_ghost_ent_ids.end() );
 PCHECK( expected_ghost_ent_ids == actual_ghost_ent_ids );
 
 // check we only have the partitioned and ghosted entities
 // on this processor.
 Range myents;
 for( Range::iterator i = partition_geom[3].begin(); i != partition_geom[3].end(); ++i )
 {
 ents.clear();
 rval = moab.get_entities_by_dimension( *i, 3, ents );CHKERR( rval );
 myents.merge( ents );
 }
 if( ghost_dimension != 3 )
 {
 ents.clear();
 rval = moab.get_adjacencies( myents, ghost_dimension, false, ents, Interface::UNION );
 PCHECK( MB_SUCCESS == rval );
 myents.swap( ents );
 }
 myents.merge( ghost_ents );
 ents.clear();
 rval = moab.get_entities_by_dimension( 0, ghost_dimension, ents );
 PCHECK( ents == myents );
 
 rval = pcomm->check_all_shared_handles();
 if( MB_SUCCESS != rval ) error = 1;
 PCHECK( !error );
 
 // done
 return MB_SUCCESS;
}
 
ErrorCode test_ghost_elements_3_2_1( const char* filename )
{
 return test_ghost_elements( filename, 3, 2, 1 );
}
 
ErrorCode test_ghost_elements_3_2_2( const char* filename )
{
 return test_ghost_elements( filename, 3, 2, 2 );
}
 
ErrorCode test_ghost_elements_3_0_1( const char* filename )
{
 return test_ghost_elements( filename, 3, 0, 1 );
}
 
ErrorCode test_ghost_elements_2_0_1( const char* filename )
{
 return test_ghost_elements( filename, 2, 0, 1 );
}
 
ErrorCode get_owner_handles( ParallelComm* pcomm, const Range& ents, EntityHandle* handle_arr )
{
 size_t i = 0;
 int junk;
 for( Range::iterator j = ents.begin(); j != ents.end(); ++i, ++j )
 {
 ErrorCode rval = pcomm->get_owner_handle( *j, junk, handle_arr[i] );
 if( MB_SUCCESS != rval ) return rval;
 }
 return MB_SUCCESS;
}
 
ErrorCode test_ghost_tag_exchange( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_DELETE;"
 "PARTITION=GEOM_DIMENSION;PARTITION_VAL=3;"
 "PARTITION_DISTRIBUTE;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_GHOSTS=3.2.1;"
 "PARALLEL_SEQUENCE_FACTOR=1.5" );CHKERR( rval );
 
 // Get ghost elements
 ParallelComm* pcomm = ParallelComm::get_pcomm( &moab, 0 );
 Range local, ghosts;
 rval = moab.get_entities_by_dimension( 0, 3, local );CHKERR( rval );
 Range::iterator i = local.begin();
 while( i != local.end() )
 {
 int rank;
 rval = pcomm->get_owner( *i, rank );CHKERR( rval );
 if( rank == (int)pcomm->proc_config().proc_rank() )
 {
 ++i;
 }
 else
 {
 ghosts.insert( *i );
 i = local.erase( i );
 }
 }
 
 // create a tag to exchange
 Tag dense_test_tag;
 EntityHandle defval = 0;
 // rval = moab.tag_get_handle( "TEST-TAG", sizeof(EntityHandle), MB_TAG_DENSE,
 // dense_test_tag, &defval ); CHKERR(rval);
 rval = moab.tag_get_handle( "TEST-TAG", 1, MB_TYPE_HANDLE, dense_test_tag, MB_TAG_DENSE | MB_TAG_EXCL, &defval );CHKERR( rval );
 
 // for all entities that I own, set tag to handle value
 std::vector< EntityHandle > handles( local.size() ), handles2;
 std::copy( local.begin(), local.end(), handles.begin() );
 rval = moab.tag_set_data( dense_test_tag, local, &handles[0] );CHKERR( rval );
 
 // exchange tag data
 Range tmp_range;
 rval = pcomm->exchange_tags( dense_test_tag, tmp_range );CHKERR( rval );
 
 // make sure local values are unchanged
 handles2.resize( local.size() );
 rval = moab.tag_get_data( dense_test_tag, local, &handles2[0] );CHKERR( rval );
 PCHECK( handles == handles2 );
 
 // compare values on ghost entities
 handles.resize( ghosts.size() );
 handles2.resize( ghosts.size() );
 rval = moab.tag_get_data( dense_test_tag, ghosts, &handles2[0] );CHKERR( rval );
 rval = get_owner_handles( pcomm, ghosts, &handles[0] );CHKERR( rval );
 PCHECK( handles == handles2 );
 
 // now do it all again for a sparse tag
 Tag sparse_test_tag;
 // rval = moab.tag_get_handle( "TEST-TAG-2", sizeof(int), MB_TAG_DENSE,
 // MB_TYPE_INTEGER, sparse_test_tag, 0 ); CHKERR(rval);
 rval = moab.tag_get_handle( "TEST-TAG-2", 1, MB_TYPE_INTEGER, sparse_test_tag, MB_TAG_DENSE | MB_TAG_EXCL );CHKERR( rval );
 
 // for all entiites that I own, set tag to my rank
 std::vector< int > procs1( local.size(), pcomm->proc_config().proc_rank() );
 rval = moab.tag_set_data( sparse_test_tag, local, &procs1[0] );CHKERR( rval );
 
 // exchange tag data
 tmp_range.clear();
 rval = pcomm->exchange_tags( sparse_test_tag, tmp_range );
 PCHECK( MB_SUCCESS == rval );
 
 // make sure local values are unchanged
 std::vector< int > procs2( local.size() );
 rval = moab.tag_get_data( sparse_test_tag, local, &procs2[0] );CHKERR( rval );
 PCHECK( procs1 == procs2 );
 
 // compare values on ghost entities
 procs1.resize( ghosts.size() );
 procs2.resize( ghosts.size() );
 rval = moab.tag_get_data( sparse_test_tag, ghosts, &procs2[0] );CHKERR( rval );
 std::vector< int >::iterator j = procs1.begin();
 for( i = ghosts.begin(); i != ghosts.end(); ++i, ++j )
 {
 rval = pcomm->get_owner( *i, *j );CHKERR( rval );
 }
 PCHECK( procs1 == procs2 );
 
 return MB_SUCCESS;
}
 
ErrorCode regression_ghost_tag_exchange_no_default( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
#ifdef MOAB_HAVE_HDF5
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_DELETE;"
 "PARTITION=GEOM_DIMENSION;PARTITION_VAL=3;"
 "PARTITION_DISTRIBUTE;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_GHOSTS=3.2.1" );
#else
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_BCAST;"
 "PARTITION=GEOM_DIMENSION;PARTITION_VAL=3;"
 "PARTITION_DISTRIBUTE;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_GHOSTS=3.2.1" );
#endif
 CHKERR( rval );
 
 // create a tag to exchange
 Tag dense_test_tag;
 // rval = moab.tag_get_handle( "TEST-TAG", sizeof(EntityHandle), MB_TAG_DENSE,
 // dense_test_tag, 0 ); CHKERR(rval);
 rval = moab.tag_get_handle( "TEST-TAG", 1, MB_TYPE_HANDLE, dense_test_tag, MB_TAG_DENSE | MB_TAG_EXCL );CHKERR( rval );
 
 // exchange tag data
 ParallelComm* pcomm = ParallelComm::get_pcomm( &moab, 0 );
 Range tmp_range;
 rval = pcomm->exchange_tags( dense_test_tag, tmp_range );
 PCHECK( MB_SUCCESS == rval );
 
 return MB_SUCCESS;
}
 
// Helper for exhange_sharing_data
// Swap contens of buffer with specified processor.
int MPI_swap( void* buffer, int num_val, MPI_Datatype val_type, int other_proc )
{
 int err, rank, bytes;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Type_size( val_type, &bytes );
 bytes *= num_val;
 std::vector< unsigned char > buffer2( bytes );
 
 for( int i = 0; i < 2; ++i )
 {
 if( i == ( rank < other_proc ) )
 {
 err = MPI_Send( buffer, num_val, val_type, other_proc, 0, MPI_COMM_WORLD );
 if( err ) return err;
 }
 else
 {
 MPI_Status status;
 err = MPI_Recv( &buffer2[0], num_val, val_type, other_proc, 0, MPI_COMM_WORLD, &status );
 if( err ) return err;
 }
 }
 
 memcpy( buffer, &buffer2[0], bytes );
 return 0;
}
 
int valid_ghosting_owners( int comm_size, const int* ids, const int* owners )
{
 // for each vertex ID, build list of {rank,owner} tuples
 std::map< int, std::vector< int > > verts;
 for( int p = 0; p < comm_size; ++p )
 {
 for( int i = 0; i < 9; ++i )
 { // nine verts per proc
 int idx = 9 * p + i;
 verts[ids[idx]].push_back( p );
 verts[ids[idx]].push_back( owners[idx] );
 }
 }
 
 // now check for each vertex that the owner from
 // each processor is the same
 bool print_desc = true;
 int error_count = 0;
 std::map< int, std::vector< int > >::iterator it;
 for( it = verts.begin(); it != verts.end(); ++it )
 {
 int id = it->first;
 std::vector< int >& list = it->second;
 bool all_same = true;
 for( size_t i = 2; i < list.size(); i += 2 )
 if( list[i + 1] != list[1] ) all_same = false;
 if( all_same ) continue;
 
 ++error_count;
 
 if( print_desc )
 {
 print_desc = false;
 std::cerr << "ERROR at " __FILE__ ":" << __LINE__ << std::endl
 << " Processors have inconsistant ideas of vertex ownership:" << std::endl;
 }
 
 std::cerr << " Vertex " << id << ": " << std::endl;
 for( size_t i = 0; i < list.size(); i += 2 )
 std::cerr << " Proc " << list[i] << " thinks owner is " << list[i + 1] << std::endl;
 }
 
 return error_count;
}
 
ErrorCode test_interface_owners_common( int num_ghost_layers )
{
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 // build distributed quad mesh
 Range quads;
 EntityHandle verts[9];
 int ids[9];
 rval = parallel_create_mesh( mb, ids, verts, quads );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quads, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 if( num_ghost_layers )
 {
 rval = pcomm.exchange_ghost_cells( 2, 0, num_ghost_layers, 0, true );
 PCHECK( MB_SUCCESS == rval );
 }
 
 // get vertex owners
 int owner[9];
 for( int i = 0; i < 9; ++i )
 {
 rval = pcomm.get_owner( verts[i], owner[i] );
 if( MB_SUCCESS != rval ) break;
 }
 PCHECK( MB_SUCCESS == rval );
 
 // exchange vertex owners amongst all processors
 int rank, size, ierr;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 std::vector< int > all_ids( 9 * size ), all_owner( 9 * size );
 ierr = MPI_Gather( ids, 9, MPI_INT, &all_ids[0], 9, MPI_INT, 0, MPI_COMM_WORLD );
 if( ierr ) return MB_FAILURE;
 ierr = MPI_Gather( owner, 9, MPI_INT, &all_owner[0], 9, MPI_INT, 0, MPI_COMM_WORLD );
 if( ierr ) return MB_FAILURE;
 
 int errors = rank ? 0 : valid_ghosting_owners( size, &all_ids[0], &all_owner[0] );
 MPI_Bcast( &errors, 1, MPI_INT, 0, MPI_COMM_WORLD );
 return errors ? MB_FAILURE : MB_SUCCESS;
}
 
// Common implementation for both:
// test_interface
// regression_interface_with_ghosting
ErrorCode test_interface_owners( const char* )
{
 return test_interface_owners_common( 0 );
}
 
ErrorCode regression_owners_with_ghosting( const char* )
{
 return test_interface_owners_common( 1 );
}
 
struct VtxData
{
 std::vector< int > procs;
 std::vector< int > owners;
 std::vector< EntityHandle > handles;
};
 
ErrorCode test_ghosted_entity_shared_data( const char* )
{
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank, size;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 // build distributed quad mesh
 Range quads;
 EntityHandle verts[9];
 int ids[9];
 rval = parallel_create_mesh( mb, ids, verts, quads );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quads, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.exchange_ghost_cells( 2, 1, 1, 0, true );
 PCHECK( MB_SUCCESS == rval );
 
 rval = pcomm.check_all_shared_handles();
 PCHECK( MB_SUCCESS == rval );
 
 return MB_SUCCESS;
}
 
ErrorCode check_consistent_ids( Interface& mb,
 const EntityHandle* entities,
 const int* orig_ids,
 int num_ents,
 const char* singular_name,
 const char* plural_name )
{
 ErrorCode rval;
 int rank, size, ierr;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 Tag id_tag = mb.globalId_tag();
 std::vector< int > new_ids( num_ents );
 rval = mb.tag_get_data( id_tag, entities, num_ents, &new_ids[0] );CHKERR( rval );
 // This test is wrong. a) The caller can select a start ID so there's
 // no guarantee that the IDs will be in any specific range and b) There
 // is no reason to expect the global number of entities to be num_ents*size
 // if there are any shared entities. J.Kraftcheck 2010-12-22
 // rval = MB_SUCCESS;
 // for (int i = 0; i < num_ents; ++i)
 // if (new_ids[i] < 0 || new_ids[i] >= num_ents*size) {
 // std::cerr << "ID out of bounds on proc " << rank
 // << " : " << new_ids[i] << " not in [0," << num_ents*size-1
 // << "]" << std::endl;
 // rval = MB_FAILURE;
 // }
 // if (MB_SUCCESS != rval)
 // return rval;
 
 // Gather up all data on root proc for consistency check
 std::vector< int > all_orig_ids( num_ents * size ), all_new_ids( num_ents * size );
 ierr = MPI_Gather( (void*)orig_ids, num_ents, MPI_INT, &all_orig_ids[0], num_ents, MPI_INT, 0, MPI_COMM_WORLD );
 if( ierr ) return MB_FAILURE;
 ierr = MPI_Gather( &new_ids[0], num_ents, MPI_INT, &all_new_ids[0], num_ents, MPI_INT, 0, MPI_COMM_WORLD );
 if( ierr ) return MB_FAILURE;
 
 // build a local map from original ID to new ID and use it
 // to check for consistancy between all procs
 rval = MB_SUCCESS;
 ;
 if( 0 == rank )
 {
 // check for two processors having different global ID for same entity
 std::map< int, int > idmap; // index by original ID and contains new ID
 std::map< int, int > owner; // index by original ID and contains owning rank
 for( int i = 0; i < num_ents * size; ++i )
 {
 std::map< int, int >::iterator it = idmap.find( all_orig_ids[i] );
 if( it == idmap.end() )
 {
 idmap[all_orig_ids[i]] = all_new_ids[i];
 owner[all_orig_ids[i]] = i / num_ents;
 }
 else if( it->second != all_new_ids[i] )
 {
 std::cerr << "Inconsistant " << singular_name << " IDs between processors " << owner[all_orig_ids[i]]
 << " and " << i / num_ents << " : " << it->second << " and " << all_new_ids[i]
 << " respectively." << std::endl;
 rval = MB_FAILURE;
 }
 }
 // check for two processors having same global ID for different entities
 idmap.clear();
 owner.clear();
 for( int i = 0; i < num_ents * size; ++i )
 {
 std::map< int, int >::iterator it = idmap.find( all_new_ids[i] );
 if( it == idmap.end() )
 {
 idmap[all_new_ids[i]] = all_orig_ids[i];
 owner[all_new_ids[i]] = i / num_ents;
 }
 else if( it->second != all_orig_ids[i] )
 {
 std::cerr << "ID " << all_new_ids[i] << " assigned to different " << plural_name << " on processors "
 << owner[all_new_ids[i]] << " and " << i / num_ents << std::endl;
 rval = MB_FAILURE;
 }
 }
 }
 return rval;
}
 
ErrorCode test_assign_global_ids( const char* )
{
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank, size;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 // build distributed quad mesh
 Range quad_range;
 EntityHandle verts[9];
 int vert_ids[9];
 rval = parallel_create_mesh( mb, vert_ids, verts, quad_range );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quad_range, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 
 // get global ids for quads
 Tag id_tag = mb.globalId_tag();
 assert( 4u == quad_range.size() );
 EntityHandle quads[4];
 std::copy( quad_range.begin(), quad_range.end(), quads );
 int quad_ids[4];
 rval = mb.tag_get_data( id_tag, quads, 4, quad_ids );CHKERR( rval );
 
 // clear GLOBAL_ID tag
 int zero[9] = { 0 };
 rval = mb.tag_set_data( id_tag, verts, 9, zero );CHKERR( rval );
 rval = mb.tag_set_data( id_tag, quads, 4, zero );CHKERR( rval );
 
 // assign new global IDs
 rval = pcomm.assign_global_ids( 0, 2 );
 PCHECK( MB_SUCCESS == rval );
 
 rval = check_consistent_ids( mb, verts, vert_ids, 9, "vertex", "vertices" );
 PCHECK( MB_SUCCESS == rval );
 rval = check_consistent_ids( mb, quads, quad_ids, 4, "quad", "quads" );
 PCHECK( MB_SUCCESS == rval );
 return rval;
}
 
ErrorCode test_shared_sets( const char* )
{
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank_i, size_i;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank_i );
 MPI_Comm_size( MPI_COMM_WORLD, &size_i );
 const unsigned rank = rank_i;
 const unsigned nproc = size_i;
 
 // build distributed quad mesh
 Range quads, sets;
 EntityHandle verts[9], set_arr[3];
 int ids[9];
 rval = parallel_create_mesh( mb, ids, verts, quads, set_arr );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quads, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 sets.insert( set_arr[0] );
 sets.insert( set_arr[1] );
 if( set_arr[2] )
 {
 sets.insert( set_arr[2] );
 }
 else
 {
 set_arr[2] = set_arr[1];
 }
 
 Tag id_tag = mb.globalId_tag();
 rval = pcomm.resolve_shared_sets( sets, id_tag );
 PCHECK( MB_SUCCESS == rval );
 
 // check that set data is consistent
 ErrorCode ok = MB_SUCCESS;
 for( size_t i = 0; i < 3; ++i )
 {
 unsigned owner;
 EntityHandle owner_handle, local;
 rval = pcomm.get_entityset_owner( set_arr[i], owner, &owner_handle );
 if( MB_SUCCESS != rval )
 ok = rval;
 else if( owner == rank )
 {
 if( owner_handle != set_arr[i] )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank << "invalid remote handle for owned set"
 << std::endl;
 ok = MB_FAILURE;
 }
 }
 else if( MB_SUCCESS != pcomm.get_entityset_local_handle( owner, owner_handle, local ) )
 ok = MB_FAILURE;
 else if( local != set_arr[i] )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank << "invalid local handle for remote data"
 << std::endl;
 ok = MB_FAILURE;
 }
 }
 PCHECK( MB_SUCCESS == ok );
 
 const unsigned col = rank / 2; // which column is proc in
 const unsigned colrank =
 2 * col; // rank of first of two procs in col (rank if rank is even, rank-1 if rank is odd)
 unsigned mins[3] = { 0, 0, 0 };
 unsigned maxs[3] = { nproc - 1, 0, 0 };
 if( rank < 2 )
 { // if in first (left-most) column, then
 mins[1] = mins[2] = 0;
 maxs[1] = maxs[2] = std::min( 3u, nproc - 1 );
 }
 else if( col == ( nproc - 1 ) / 2 )
 { // else if last (right-most) column, then
 mins[1] = mins[2] = colrank - 2;
 maxs[1] = maxs[2] = std::min( colrank + 1, nproc - 1 );
 }
 else
 { // if inside column, then
 mins[1] = colrank - 2;
 mins[2] = colrank;
 maxs[1] = std::min( colrank + 1, nproc - 1 );
 maxs[2] = std::min( colrank + 3, nproc - 1 );
 }
 
 // check expected sharing lists
 // set 0 is shared between all procs in the range [ mins[0], maxs[0] ]
 std::vector< unsigned > expected, list;
 for( size_t i = 0; i < 3; ++i )
 {
 expected.clear();
 for( unsigned r = mins[i]; r <= std::min( maxs[i], nproc - 1 ); ++r )
 if( r != rank ) expected.push_back( r );
 list.clear();
 rval = pcomm.get_entityset_procs( set_arr[i], list );
 if( MB_SUCCESS != rval )
 ok = rval;
 else
 {
 std::sort( list.begin(), list.end() );
 if( expected != list )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank << " incorrect sharing list for entity set"
 << std::endl;
 ok = MB_FAILURE;
 }
 }
 }
 PCHECK( MB_SUCCESS == ok );
 
 // check consistent owners
 unsigned send_list[6], set_owners[3];
 std::vector< unsigned > recv_list( 6 * nproc );
 for( size_t i = 0; i < 3; ++i )
 {
 mb.tag_get_data( id_tag, set_arr + i, 1, &send_list[2 * i] );
 pcomm.get_entityset_owner( set_arr[i], set_owners[i] );
 send_list[2 * i + 1] = set_owners[i];
 }
 MPI_Allgather( send_list, 6, MPI_UNSIGNED, &recv_list[0], 6, MPI_UNSIGNED, MPI_COMM_WORLD );
 std::map< unsigned, unsigned > owners;
 for( unsigned i = 0; i < 6 * nproc; i += 2 )
 {
 unsigned id = recv_list[i];
 unsigned owner = recv_list[i + 1];
 if( owners.find( id ) == owners.end() )
 owners[id] = owner;
 else if( owners[id] != owner )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank << " mismatched owners (" << owners[id]
 << " and " << owner << ") for set with ID " << id << std::endl;
 ok = MB_FAILURE;
 }
 }
 PCHECK( MB_SUCCESS == ok );
 
 // test PComm::get_entityset_owners
 std::vector< unsigned > act_owners, exp_owners;
 for( size_t i = 0; i < 3; ++i )
 {
 exp_owners.push_back( set_owners[i] );
 }
 ok = pcomm.get_entityset_owners( act_owners );
 PCHECK( MB_SUCCESS == ok );
 // PCHECK(std::find(act_owners.begin(),act_owners.end(),rank) == act_owners.end());
 std::sort( act_owners.begin(), act_owners.end() );
 std::sort( exp_owners.begin(), exp_owners.end() );
 exp_owners.erase( std::unique( exp_owners.begin(), exp_owners.end() ), exp_owners.end() );
 PCHECK( exp_owners == act_owners );
 
 // test PComm::get_owned_sets
 ok = MB_SUCCESS;
 for( unsigned i = 0; i < nproc; ++i )
 {
 sets.clear();
 if( MB_SUCCESS != pcomm.get_owned_sets( i, sets ) )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank
 << " failed to get shared set list for sets owned by rank " << set_owners[i] << std::endl;
 continue;
 }
 
 Range expected_range;
 for( size_t j = 0; j < 3; ++j )
 if( set_owners[j] == i ) expected_range.insert( set_arr[j] );
 
 if( expected_range != sets )
 {
 std::cerr << __FILE__ << ":" << __LINE__ << " rank " << rank
 << " has incorrect shared set list for sets owned by rank " << set_owners[i] << std::endl
 << "Expected: " << expected_range << std::endl
 << "Actual: " << sets << std::endl;
 ok = MB_FAILURE;
 }
 }
 PCHECK( MB_SUCCESS == ok );
 
 return MB_SUCCESS;
}
 
template < typename T >
ErrorCode check_shared_ents( ParallelComm& pcomm, Tag tagh, T fact, MPI_Op mpi_op )
{
 // get the shared entities
 Range shared_ents;
 ErrorCode rval = pcomm.get_shared_entities( -1, shared_ents );CHKERR( rval );
 
 std::vector< int > shprocs( MAX_SHARING_PROCS );
 std::vector< EntityHandle > shhandles( MAX_SHARING_PROCS );
 std::vector< T > dum_vals( shared_ents.size() );
 int np;
 unsigned char pstatus;
 
 rval = pcomm.get_moab()->tag_get_data( tagh, shared_ents, &dum_vals[0] );CHKERR( rval );
 
 // for each, compare number of sharing procs against tag value, should be 1/fact the tag value
 Range::iterator rit;
 int i = 0;
 for( rit = shared_ents.begin(); rit != shared_ents.end(); ++rit, i++ )
 {
 rval = pcomm.get_sharing_data( *rit, &shprocs[0], &shhandles[0], pstatus, np );CHKERR( rval );
 if( 1 == np && shprocs[0] != (int)pcomm.proc_config().proc_rank() ) np++;
 bool with_root = std::find( &shprocs[0], &shprocs[np], 0 ) - &shprocs[0] != np || !pcomm.rank();
 if( mpi_op == MPI_SUM )
 {
 if( dum_vals[i] != fact * np ) return MB_FAILURE;
 }
 else if( mpi_op == MPI_PROD )
 {
 if( dum_vals[i] != pow( fact, np ) ) return MB_FAILURE;
 }
 else if( mpi_op == MPI_MAX )
 {
 if( with_root && dum_vals[i] != fact ) return MB_FAILURE;
 }
 else if( mpi_op == MPI_MIN )
 {
 if( with_root && dum_vals[i] != fact ) return MB_FAILURE;
 }
 else
 return MB_FAILURE;
 }
 
 return MB_SUCCESS;
}
 
template < typename T >
ErrorCode test_reduce_tags( const char*, DataType tp )
{
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank, size;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 // build distributed quad mesh
 Range quad_range;
 EntityHandle verts[9];
 int vert_ids[9];
 rval = parallel_create_mesh( mb, vert_ids, verts, quad_range );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quad_range, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 
 Tag test_tag;
 T def_val = 2;
 Range dum_range;
 std::vector< T > dum_vals;
 
 // T, MPI_SUM
 rval = mb.tag_get_handle( "test_tag", 1, tp, test_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_val );CHKERR( rval );
 rval = pcomm.reduce_tags( test_tag, MPI_SUM, dum_range );CHKERR( rval );
 rval = check_shared_ents( pcomm, test_tag, (T)2, MPI_SUM );CHKERR( rval );
 rval = mb.tag_delete( test_tag );
 
 // T, MPI_PROD
 rval = mb.tag_get_handle( "test_tag", 1, tp, test_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_val );CHKERR( rval );
 rval = pcomm.reduce_tags( test_tag, MPI_PROD, dum_range );CHKERR( rval );
 rval = check_shared_ents( pcomm, test_tag, (T)2, MPI_PROD );CHKERR( rval );
 rval = mb.tag_delete( test_tag );
 
 // T, MPI_MAX
 rval = mb.tag_get_handle( "test_tag", 1, tp, test_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_val );CHKERR( rval );
 // get owned entities and set a different value on them
 rval = pcomm.get_shared_entities( -1, dum_range, -1, false, false );CHKERR( rval );
 if( pcomm.rank() == 0 )
 {
 dum_vals.resize( dum_range.size(), (T)3 );
 rval = mb.tag_set_data( test_tag, dum_range, &dum_vals[0] );CHKERR( rval );
 }
 rval = pcomm.reduce_tags( test_tag, MPI_MAX, dum_range );CHKERR( rval );
 rval = check_shared_ents( pcomm, test_tag, (T)3, MPI_MAX );CHKERR( rval );
 rval = mb.tag_delete( test_tag );
 
 // T, MPI_MIN
 rval = mb.tag_get_handle( "test_tag", 1, tp, test_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_val );CHKERR( rval );
 // get owned entities and set a different value on them
 if( pcomm.rank() == 0 )
 {
 std::fill( dum_vals.begin(), dum_vals.end(), (T)-1 );
 rval = mb.tag_set_data( test_tag, dum_range, &dum_vals[0] );CHKERR( rval );
 }
 rval = pcomm.reduce_tags( test_tag, MPI_MIN, dum_range );CHKERR( rval );
 rval = check_shared_ents( pcomm, test_tag, (T)-1, MPI_MIN );CHKERR( rval );
 rval = mb.tag_delete( test_tag );
 
 return MB_SUCCESS;
}
 
ErrorCode test_reduce_tag_failures( const char* )
{
 // test things that should fail
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank, size;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 // build distributed quad mesh
 Range quad_range;
 EntityHandle verts[9];
 int vert_ids[9];
 rval = parallel_create_mesh( mb, vert_ids, verts, quad_range );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quad_range, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 
 Tag test_tag;
 Range dum_range;
 int def_int;
 double def_dbl;
 
 // explicit destination tag of different type
 Tag test_dest;
 std::vector< Tag > src_tags, dest_tags;
 rval = mb.tag_get_handle( "test_tag", 1, MB_TYPE_INTEGER, test_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_int );CHKERR( rval );
 src_tags.push_back( test_tag );
 rval = mb.tag_get_handle( "dtest_tag", 1, MB_TYPE_DOUBLE, test_dest, MB_TAG_DENSE | MB_TAG_CREAT, &def_dbl );CHKERR( rval );
 dest_tags.push_back( test_dest );
 rval = pcomm.reduce_tags( src_tags, dest_tags, MPI_MIN, dum_range );
 PCHECK( rval == MB_FAILURE );
 rval = mb.tag_delete( test_dest );CHKERR( rval );
 rval = mb.tag_delete( test_tag );CHKERR( rval );
 
 // tag w/ no default value
 rval = mb.tag_get_handle( "test_tag", 1, MB_TYPE_INTEGER, test_tag, MB_TAG_DENSE | MB_TAG_CREAT );CHKERR( rval );
 rval = pcomm.reduce_tags( test_tag, MPI_MIN, dum_range );
 PCHECK( rval == MB_ENTITY_NOT_FOUND );
 rval = mb.tag_delete( test_tag );
 
 return MB_SUCCESS;
}
 
ErrorCode test_reduce_tags( const char* )
{
 ErrorCode rval = MB_SUCCESS, tmp_rval;
 
 tmp_rval = test_reduce_tags< int >( "test_reduce_tags (int)", MB_TYPE_INTEGER );
 if( MB_SUCCESS != tmp_rval )
 {
 std::cout << "test_reduce_tags failed for int data type." << std::endl;
 rval = tmp_rval;
 }
 
 tmp_rval = test_reduce_tags< double >( "test_reduce_tags (dbl)", MB_TYPE_DOUBLE );
 if( MB_SUCCESS != tmp_rval )
 {
 std::cout << "test_reduce_tags failed for double data type." << std::endl;
 rval = tmp_rval;
 }
 
 return rval;
}
 
ErrorCode test_reduce_tag_explicit_dest( const char* )
{
 // test tag_reduce stuff with explicit destination tag
 ErrorCode rval;
 Core moab_instance;
 Interface& mb = moab_instance;
 ParallelComm pcomm( &mb, MPI_COMM_WORLD );
 
 int rank, size;
 MPI_Comm_rank( MPI_COMM_WORLD, &rank );
 MPI_Comm_size( MPI_COMM_WORLD, &size );
 
 // build distributed quad mesh
 Range quad_range;
 EntityHandle verts[9];
 int vert_ids[9];
 rval = parallel_create_mesh( mb, vert_ids, verts, quad_range );
 PCHECK( MB_SUCCESS == rval );
 rval = pcomm.resolve_shared_ents( 0, quad_range, 2, 1 );
 PCHECK( MB_SUCCESS == rval );
 
 Tag src_tag, dest_tag;
 Range dum_range;
 double def_dbl = 5.0;
 
 // src tag has one value, pre-existing dest tag has another; should get MPI_Op of src values
 std::vector< Tag > src_tags, dest_tags;
 // create the tags
 rval = mb.tag_get_handle( "src_tag", 1, MB_TYPE_DOUBLE, src_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_dbl );CHKERR( rval );
 src_tags.push_back( src_tag );
 rval = mb.tag_get_handle( "dest_tag", 1, MB_TYPE_DOUBLE, dest_tag, MB_TAG_DENSE | MB_TAG_CREAT, &def_dbl );CHKERR( rval );
 dest_tags.push_back( dest_tag );
 // set values for src tag to one value, dest tag to another
 rval = pcomm.get_shared_entities( -1, dum_range, -1, false, false );CHKERR( rval );
 std::vector< double > tag_vals( dum_range.size(), 1.0 );
 rval = mb.tag_set_data( src_tag, dum_range, &tag_vals[0] );CHKERR( rval );
 std::fill( tag_vals.begin(), tag_vals.end(), 2.0 );
 rval = mb.tag_set_data( dest_tag, dum_range, &tag_vals[0] );CHKERR( rval );
 // do the reduce
 rval = pcomm.reduce_tags( src_tags, dest_tags, MPI_SUM, dum_range );CHKERR( rval );
 // check the reduced value
 rval = check_shared_ents< double >( pcomm, dest_tag, (double)1.0, MPI_SUM );CHKERR( rval );
 
 rval = mb.tag_delete( src_tag );CHKERR( rval );
 rval = mb.tag_delete( dest_tag );CHKERR( rval );
 
 return MB_SUCCESS;
}
 
ErrorCode test_delete_entities( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_PART;"
 "PARTITION=PARALLEL_PARTITION;"
 "PARALLEL_RESOLVE_SHARED_ENTS" );CHKERR( rval );
 
 // Get ghost elements
 ParallelComm* pcomm = ParallelComm::get_pcomm( &moab, 0 );
 // get some edges and faces, and delete some
 Range local;
 rval = moab.get_entities_by_dimension( 0, 1, local );CHKERR( rval );
 rval = moab.get_entities_by_dimension( 0, 2, local );CHKERR( rval ); // it is appending to range
 
 Range local2; // extract about half of those
 std::copy( local.begin(), local.begin() + local.size() / 2, range_inserter( local2 ) );
 
 // delete local 2
 rval = pcomm->delete_entities( local2 );CHKERR( rval );
 
 for( Range::iterator it = local2.begin(); it != local2.end(); ++it )
 {
 if( mb_instance.is_valid( *it ) ) return MB_FAILURE;
 }
 const char* opt = "PARALLEL=WRITE_PART";
 rval = moab.write_file( "tmpx.h5m", 0, opt );CHKERR( rval );
 if( pcomm->proc_config().proc_rank() == 0 ) remove( "tmpx.h5m" );
 
 return MB_SUCCESS;
}
 
ErrorCode test_ghost_polyhedra( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_PART;"
 "PARTITION=PARALLEL_PARTITION;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_GHOSTS=3.0.1" );CHKERR( rval );
 
 return MB_SUCCESS;
}
ErrorCode test_too_few_parts( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_PART;"
 "PARTITION=PARALLEL_PARTITION;"
 "PARALLEL_RESOLVE_SHARED_ENTS;" );
 // if(rval==MB_SUCCESS)
 // return MB_FAILURE;
 
 return rval;
}
 
ErrorCode test_sequences_after_ghosting( const char* filename )
{
 Core mb_instance;
 Interface& moab = mb_instance;
 ErrorCode rval;
 
 rval = moab.load_file( filename, 0,
 "PARALLEL=READ_PART;"
 "PARTITION=PARALLEL_PARTITION;"
 "PARALLEL_RESOLVE_SHARED_ENTS;"
 "PARALLEL_GHOSTS=3.2.1;"
 "PARALLEL_SEQUENCE_FACTOR=1.5" );CHKERR( rval );
 
 // get all elements of dimension 3, and check they are on one sequence, with connect_iterate
 Range elems;
 rval = moab.get_entities_by_dimension( 0, 3, elems );CHKERR( rval );
 if( elems.psize() != 1 )
 {
 std::cout << " elems.psize() = " << elems.psize() << "\n";
 return MB_FAILURE;
 }
 // we want only one sequence
 int count, vpere;
 EntityHandle* conn_ptr;
 rval = moab.connect_iterate( elems.begin(), elems.end(), conn_ptr, vpere, count );CHKERR( rval );
 
 if( count != (int)elems.size() )
 {
 std::cout << " more than one sequence: elems.size() = " << elems.size() << " count:" << count << "\n";
 return MB_FAILURE;
 }
 // check also global id tag, which is dense
 Tag id_tag = moab.globalId_tag();
 void* globalid_data = NULL;
 rval = moab.tag_iterate( id_tag, elems.begin(), elems.end(), count, globalid_data );CHKERR( rval );
 if( count != (int)elems.size() )
 {
 std::cout << " more than one tag sequence: elems.size() = " << elems.size() << " count:" << count << "\n";
 return MB_FAILURE;
 }
 
 // repeat the tests for vertex sequences
 // get all elements of dimension 3, and check they are on one sequence, with coords_iterate
 Range verts;
 rval = moab.get_entities_by_dimension( 0, 0, verts );CHKERR( rval );
 if( verts.psize() != 1 )
 {
 std::cout << " verts.psize() = " << verts.psize() << "\n";
 return MB_FAILURE;
 }
 //
 double *x_ptr, *y_ptr, *z_ptr;
 rval = moab.coords_iterate( verts.begin(), verts.end(), x_ptr, y_ptr, z_ptr, count );CHKERR( rval );
 
 if( count != (int)verts.size() )
 {
 std::cout << " more than one sequence: verts.size() = " << verts.size() << " count:" << count << "\n";
 return MB_FAILURE;
 }
 
 rval = moab.tag_iterate( id_tag, verts.begin(), verts.end(), count, globalid_data );CHKERR( rval );
 if( count != (int)verts.size() )
 {
 std::cout << " more than one tag sequence: verts.size() = " << verts.size() << " count:" << count << "\n";
 return MB_FAILURE;
 }
 return MB_SUCCESS;
}
 
// test trivial partition as used by iMOAB send/receive mesh methods
// it is hooked to ParCommGraph, but it is really not dependent on anything from MOAB
// these methods that are tested are just utilities
 
void test_trivial_partition()
{
 // nothing is modified inside moab
 // by these methods;
 // to instantiate par com graph, we just need 2 groups!
 // they can be overlapping !!!! we do not test that yet
 
 // create 2 groups; one over task 0 and 1, one over 2
 MPI_Group worldg;
 MPI_Comm duplicate;
 MPI_Comm_dup( MPI_COMM_WORLD, &duplicate );
 MPI_Comm_group( duplicate, &worldg );
 
 // this is usually run on 2 processes
 
 int rank[2] = { 0, 1 };
 MPI_Group gr1, gr2;
 MPI_Group_incl( worldg, 2, rank, &gr1 ); // will contain 2 ranks, 0 and 1
 MPI_Group_incl( worldg, 1, &rank[1], &gr2 ); // will contain 1 rank only (1)
 
 int comp1 = 10, comp2 = 12;
 ParCommGraph* pgr = new ParCommGraph( duplicate, gr1, gr2, comp1, comp2 );
 
 std::map< int, Range > ranges_to_send;
 std::vector< int > number_elems_per_part;
 number_elems_per_part.push_back( 6 );
 number_elems_per_part.push_back( 10 );
 
 std::cout << " send sizes ";
 for( int k = 0; k < (int)number_elems_per_part.size(); k++ )
 {
 std::cout << " " << number_elems_per_part[k];
 }
 std::cout << "\n";
 
 std::cout << "\n";
 pgr->compute_trivial_partition( number_elems_per_part );
 
 Range verts( 10, 20 );
 pgr->split_owned_range( 0, verts );
 
 for( std::map< int, Range >::iterator it = ranges_to_send.begin(); it != ranges_to_send.end(); it++ )
 {
 Range& ran = it->second;
 std::cout << " receiver " << it->first << " receive range: [" << ran[0] << ", " << ran[ran.size() - 1]
 << "] \n";
 }
 
 delete( pgr );
 
 MPI_Group_free( &worldg );
 MPI_Group_free( &gr1 );
 MPI_Group_free( &gr2 );
 MPI_Comm_free( &duplicate );
}