ParallelComm.cpp

Go to the documentation of this file.

#include "moab/Interface.hpp"
#include "moab/ParallelComm.hpp"
#include "moab/WriteUtilIface.hpp"
#include "moab/ReadUtilIface.hpp"
#include "SequenceManager.hpp"
#include "moab/Error.hpp"
#include "EntitySequence.hpp"
#include "MBTagConventions.hpp"
#include "moab/Skinner.hpp"
#include "MBParallelConventions.h"
#include "moab/Core.hpp"
#include "ElementSequence.hpp"
#include "moab/CN.hpp"
#include "moab/RangeMap.hpp"
#include "moab/MeshTopoUtil.hpp"
#include "TagInfo.hpp"
#include "DebugOutput.hpp"
#include "SharedSetData.hpp"
#include "moab/ScdInterface.hpp"
#include "moab/TupleList.hpp"
#include "moab/gs.hpp"
 
#include <iostream>
#include <sstream>
#include <algorithm>
#include <functional>
#include <numeric>
 
#include <cmath>
#include <cstdlib>
#include <cassert>
 
#ifdef MOAB_HAVE_MPI
#include "moab_mpi.h"
#endif
#ifdef MOAB_HAVE_MPE
#include "mpe.h"
int IFACE_START, IFACE_END;
int GHOST_START, GHOST_END;
int SHAREDV_START, SHAREDV_END;
int RESOLVE_START, RESOLVE_END;
int ENTITIES_START, ENTITIES_END;
int RHANDLES_START, RHANDLES_END;
int OWNED_START, OWNED_END;
#endif
 
namespace moab
{
 
const unsigned int ParallelComm::INITIAL_BUFF_SIZE = 1024;
 
const int MAX_BCAST_SIZE = ( 1 << 28 );
 
std::vector< ParallelComm::Buffer* > msgs;
unsigned int __PACK_num = 0, __UNPACK_num = 0, __PACK_count = 0, __UNPACK_count = 0;
std::string __PACK_string, __UNPACK_string;
 
#ifdef DEBUG_PACKING_TIMES
#define PC( n, m )                                                            \
    {                                                                         \
        if( __PACK_num == (unsigned int)n && __PACK_string == m )             \
            __PACK_count++;                                                   \
        else                                                                  \
        {                                                                     \
            if( __PACK_count > 1 ) std::cerr << " (" << __PACK_count << "x)"; \
            __PACK_count  = 1;                                                \
            __PACK_string = m;                                                \
            __PACK_num    = n;                                                \
            std::cerr << std::endl << "PACK: " << n << m;                     \
        }                                                                     \
    }
#define UPC( n, m )                                                              \
    {                                                                            \
        if( __UNPACK_num == (unsigned int)n && __UNPACK_string == m )            \
            __UNPACK_count++;                                                    \
        else                                                                     \
        {                                                                        \
            if( __UNPACK_count > 1 ) std::cerr << "(" << __UNPACK_count << "x)"; \
            __UNPACK_count  = 1;                                                 \
            __UNPACK_string = m;                                                 \
            __UNPACK_num    = n;                                                 \
            std::cerr << std::endl << "UNPACK: " << n << m;                      \
        }                                                                        \
    }
#else
#define PC( n, m )
#define UPC( n, m )
#endif
 
template < typename T >
static inline void UNPACK( unsigned char*& buff, T* val, size_t count )
{
    memcpy( val, buff, count * sizeof( T ) );
    buff += count * sizeof( T );
}
 
template < typename T >
static inline void PACK( unsigned char*& buff, const T* val, size_t count )
{
    memcpy( buff, val, count * sizeof( T ) );
    buff += count * sizeof( T );
}
 
static inline void PACK_INTS( unsigned char*& buff, const int* int_val, size_t num )
{
    PACK( buff, int_val, num );
    PC( num, " ints" );
}
 
static inline void PACK_INT( unsigned char*& buff, int int_val )
{
    PACK_INTS( buff, &int_val, 1 );
}
 
static inline void PACK_DBLS( unsigned char*& buff, const double* dbl_val, size_t num )
{
    PACK( buff, dbl_val, num );
    PC( num, " doubles" );
}
 
// static inline
// void PACK_DBL(unsigned char*& buff, const double dbl_val)
//{ PACK_DBLS(buff, &dbl_val, 1); }
 
static inline void PACK_EH( unsigned char*& buff, const EntityHandle* eh_val, size_t num )
{
    PACK( buff, eh_val, num );
    PC( num, " handles" );
}
 
// static inline
// void PACK_CHAR_64(unsigned char*& buff, const char* str)
//{
//  memcpy(buff, str, 64);
//  buff += 64;
//  PC(64, " chars");
//}
 
static inline void PACK_VOID( unsigned char*& buff, const void* val, size_t num )
{
    PACK( buff, reinterpret_cast< const unsigned char* >( val ), num );
    PC( num, " void" );
}
 
static inline void PACK_BYTES( unsigned char*& buff, const void* val, int num )
{
    PACK_INT( buff, num );
    PACK_VOID( buff, val, num );
}
 
static inline void PACK_RANGE( unsigned char*& buff, const Range& rng )
{
    PACK_INT( buff, rng.psize() );
    Range::const_pair_iterator cit;
    for( cit = rng.const_pair_begin(); cit != rng.const_pair_end(); ++cit )
    {
        EntityHandle eh[2] = { cit->first, cit->second };
        PACK_EH( buff, eh, 2 );
    }
    PC( rng.psize(), "-subranged range" );
}
 
static inline void UNPACK_INTS( unsigned char*& buff, int* int_val, size_t num )
{
    UNPACK( buff, int_val, num );
    UPC( num, " ints" );
}
 
static inline void UNPACK_INT( unsigned char*& buff, int& int_val )
{
    UNPACK_INTS( buff, &int_val, 1 );
}
 
static inline void UNPACK_DBLS( unsigned char*& buff, double* dbl_val, size_t num )
{
    UNPACK( buff, dbl_val, num );
    UPC( num, " doubles" );
}
 
static inline void UNPACK_DBL( unsigned char*& buff, double& dbl_val )
{
    UNPACK_DBLS( buff, &dbl_val, 1 );
}
 
static inline void UNPACK_EH( unsigned char*& buff, EntityHandle* eh_val, size_t num )
{
    UNPACK( buff, eh_val, num );
    UPC( num, " handles" );
}
 
// static inline
// void UNPACK_CHAR_64(unsigned char*& buff, char* char_val)
//{
//  memcpy(buff, char_val, 64);
//  buff += 64;
//  UPC(64, " chars");
//}
 
static inline void UNPACK_VOID( unsigned char*& buff, void* val, size_t num )
{
    UNPACK( buff, reinterpret_cast< unsigned char* >( val ), num );
    UPC( num, " void" );
}
 
static inline void UNPACK_TYPE( unsigned char*& buff, EntityType& type )
{
    int int_type = MBMAXTYPE;
    UNPACK_INT( buff, int_type );
    type = static_cast< EntityType >( int_type );
    assert( type >= MBVERTEX && type <= MBMAXTYPE );
}
 
static inline void UNPACK_RANGE( unsigned char*& buff, Range& rng )
{
    int num_subs;
    EntityHandle eh[2];
    UNPACK_INT( buff, num_subs );
    for( int i = 0; i < num_subs; i++ )
    {
        UPC( num_subs, "-subranged range" );
        UNPACK_EH( buff, eh, 2 );
        rng.insert( eh[0], eh[1] );
    }
}
 
enum MBMessageTag
{
    MB_MESG_ANY = MPI_ANY_TAG,
    MB_MESG_ENTS_ACK,
    MB_MESG_ENTS_SIZE,
    MB_MESG_ENTS_LARGE,
    MB_MESG_REMOTEH_ACK,
    MB_MESG_REMOTEH_SIZE,
    MB_MESG_REMOTEH_LARGE,
    MB_MESG_TAGS_ACK,
    MB_MESG_TAGS_SIZE,
    MB_MESG_TAGS_LARGE
};
 
static inline size_t RANGE_SIZE( const Range& rng )
{
    return 2 * sizeof( EntityHandle ) * rng.psize() + sizeof( int );
}
 
#define PRINT_DEBUG_ISEND( A, B, C, D, E )    print_debug_isend( ( A ), ( B ), ( C ), ( D ), ( E ) )
#define PRINT_DEBUG_IRECV( A, B, C, D, E, F ) print_debug_irecv( ( A ), ( B ), ( C ), ( D ), ( E ), ( F ) )
#define PRINT_DEBUG_RECD( A )                 print_debug_recd( ( A ) )
#define PRINT_DEBUG_WAITANY( A, B, C )        print_debug_waitany( ( A ), ( B ), ( C ) )
 
void ParallelComm::print_debug_isend( int from, int to, unsigned char* buff, int tag, int sz )
{
    myDebug->tprintf( 3, "Isend, %d->%d, buffer ptr = %p, tag=%d, size=%d\n", from, to, (void*)buff, tag, sz );
}
 
void ParallelComm::print_debug_irecv( int to, int from, unsigned char* buff, int sz, int tag, int incoming )
{
    myDebug->tprintf( 3, "Irecv, %d<-%d, buffer ptr = %p, tag=%d, size=%d", to, from, (void*)buff, tag, sz );
    if( tag < MB_MESG_REMOTEH_ACK )
        myDebug->printf( 3, ", incoming1=%d\n", incoming );
    else if( tag < MB_MESG_TAGS_ACK )
        myDebug->printf( 3, ", incoming2=%d\n", incoming );
    else
        myDebug->printf( 3, ", incoming=%d\n", incoming );
}
 
void ParallelComm::print_debug_recd( MPI_Status status )
{
    if( myDebug->get_verbosity() == 3 )
    {
        int this_count;
        int success = MPI_Get_count( &status, MPI_UNSIGNED_CHAR, &this_count );
        if( MPI_SUCCESS != success ) this_count = -1;
        myDebug->tprintf( 3, "Received from %d, count = %d, tag = %d\n", status.MPI_SOURCE, this_count,
                          status.MPI_TAG );
    }
}
 
void ParallelComm::print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc )
{
    if( myDebug->get_verbosity() == 3 )
    {
        myDebug->tprintf( 3, "Waitany, p=%d, ", proc );
        if( tag < MB_MESG_REMOTEH_ACK )
            myDebug->print( 3, ", recv_ent_reqs=" );
        else if( tag < MB_MESG_TAGS_ACK )
            myDebug->print( 3, ", recv_remoteh_reqs=" );
        else
            myDebug->print( 3, ", recv_tag_reqs=" );
        for( unsigned int i = 0; i < reqs.size(); i++ )
            myDebug->printf( 3, " %p", (void*)(intptr_t)reqs[i] );
        myDebug->print( 3, "\n" );
    }
}
 
/** Name of tag used to store ParallelComm Index on mesh paritioning sets */
const char* PARTITIONING_PCOMM_TAG_NAME = "__PRTN_PCOMM";
 
/** \brief Tag storing parallel communication objects
 *
 * This tag stores pointers to ParallelComm communication
 * objects; one of these is allocated for each different
 * communicator used to read mesh. ParallelComm stores
 * partition and interface sets corresponding to its parallel mesh.
 * By default, a parallel read uses the first ParallelComm object
 * on the interface instance; if instantiated with one, ReadParallel
 * adds this object to the interface instance too.
 *
 * Tag type: opaque
 * Tag size: MAX_SHARING_PROCS*sizeof(ParallelComm*)
 */
#define PARALLEL_COMM_TAG_NAME "__PARALLEL_COMM"
 
ParallelComm::ParallelComm( Interface* impl, MPI_Comm cm, int* id )
    : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
      pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
      myDebug( NULL )
{
    initialize();
    sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
    if( id ) *id = pcommID;
}
 
ParallelComm::ParallelComm( Interface* impl, std::vector< unsigned char >& /*tmp_buff*/, MPI_Comm cm, int* id )
    : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
      pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
      myDebug( NULL )
{
    initialize();
    sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
    if( id ) *id = pcommID;
}
 
ParallelComm::~ParallelComm()
{
    remove_pcomm( this );
    delete_all_buffers();
    delete myDebug;
    delete sharedSetData;
}
 
void ParallelComm::initialize()
{
    Core* core      = dynamic_cast< Core* >( mbImpl );
    sequenceManager = core->sequence_manager();
    mbImpl->query_interface( errorHandler );
 
    // Initialize MPI, if necessary
    int flag   = 1;
    int retval = MPI_Initialized( &flag );
    if( MPI_SUCCESS != retval || !flag )
    {
        int argc    = 0;
        char** argv = NULL;
 
        // mpi not initialized yet - initialize here
        retval = MPI_Init( &argc, &argv );
        assert( MPI_SUCCESS == retval );
    }
 
    // Reserve space for vectors
    buffProcs.reserve( MAX_SHARING_PROCS );
    localOwnedBuffs.reserve( MAX_SHARING_PROCS );
    remoteOwnedBuffs.reserve( MAX_SHARING_PROCS );
 
    pcommID = add_pcomm( this );
 
    if( !myDebug )
    {
        myDebug = new DebugOutput( "ParallelComm", std::cerr );
        myDebug->set_rank( procConfig.proc_rank() );
    }
}
 
int ParallelComm::add_pcomm( ParallelComm* pc )
{
    // Add this pcomm to instance tag
    std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS, (ParallelComm*)NULL );
    Tag pc_tag = pcomm_tag( mbImpl, true );
    assert( 0 != pc_tag );
 
    const EntityHandle root = 0;
    ErrorCode result        = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
    if( MB_SUCCESS != result && MB_TAG_NOT_FOUND != result ) return -1;
    int index = 0;
    while( index < MAX_SHARING_PROCS && pc_array[index] )
        index++;
    if( index == MAX_SHARING_PROCS )
    {
        index = -1;
        assert( false );
    }
    else
    {
        pc_array[index] = pc;
        mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
    }
    return index;
}
 
void ParallelComm::remove_pcomm( ParallelComm* pc )
{
    // Remove this pcomm from instance tag
    std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS );
    Tag pc_tag = pcomm_tag( mbImpl, true );
 
    const EntityHandle root                      = 0;
    ErrorCode result                             = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
    std::vector< ParallelComm* >::iterator pc_it = std::find( pc_array.begin(), pc_array.end(), pc );
    assert( MB_SUCCESS == result && pc_it != pc_array.end() );
    // Empty if test to get around compiler warning about unused var
    if( MB_SUCCESS == result )
    {
    }
 
    *pc_it = NULL;
    mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
}
 
//! Assign a global id space, for largest-dimension or all entities (and
//! in either case for vertices too)
ErrorCode ParallelComm::assign_global_ids( EntityHandle this_set,
                                           const int dimension,
                                           const int start_id,
                                           const bool largest_dim_only,
                                           const bool parallel,
                                           const bool owned_only )
{
    Range entities[4];
    ErrorCode result;
    std::vector< unsigned char > pstatus;
    for( int dim = 0; dim <= dimension; dim++ )
    {
        if( dim == 0 || !largest_dim_only || dim == dimension )
        {
            result = mbImpl->get_entities_by_dimension( this_set, dim, entities[dim] );MB_CHK_SET_ERR( result, "Failed to get vertices in assign_global_ids" );
        }
 
        // Need to filter out non-locally-owned entities!!!
        pstatus.resize( entities[dim].size() );
        result = mbImpl->tag_get_data( pstatus_tag(), entities[dim], &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus in assign_global_ids" );
 
        Range dum_range;
        Range::iterator rit;
        unsigned int i;
        for( rit = entities[dim].begin(), i = 0; rit != entities[dim].end(); ++rit, i++ )
            if( pstatus[i] & PSTATUS_NOT_OWNED ) dum_range.insert( *rit );
        entities[dim] = subtract( entities[dim], dum_range );
    }
 
    return assign_global_ids( entities, dimension, start_id, parallel, owned_only );
}
 
//! Assign a global id space, for largest-dimension or all entities (and
//! in either case for vertices too)
ErrorCode ParallelComm::assign_global_ids( Range entities[],
                                           const int dimension,
                                           const int start_id,
                                           const bool parallel,
                                           const bool owned_only )
{
    int local_num_elements[4];
    ErrorCode result;
    for( int dim = 0; dim <= dimension; dim++ )
    {
        local_num_elements[dim] = entities[dim].size();
    }
 
    // Communicate numbers
    std::vector< int > num_elements( procConfig.proc_size() * 4 );
#ifdef MOAB_HAVE_MPI
    if( procConfig.proc_size() > 1 && parallel )
    {
        int retval =
            MPI_Allgather( local_num_elements, 4, MPI_INT, &num_elements[0], 4, MPI_INT, procConfig.proc_comm() );
        if( 0 != retval ) return MB_FAILURE;
    }
    else
#endif
        for( int dim = 0; dim < 4; dim++ )
            num_elements[dim] = local_num_elements[dim];
 
    // My entities start at one greater than total_elems[d]
    int total_elems[4] = { start_id, start_id, start_id, start_id };
 
    for( unsigned int proc = 0; proc < procConfig.proc_rank(); proc++ )
    {
        for( int dim = 0; dim < 4; dim++ )
            total_elems[dim] += num_elements[4 * proc + dim];
    }
 
    // Assign global ids now
    Tag gid_tag = mbImpl->globalId_tag();
 
    for( int dim = 0; dim < 4; dim++ )
    {
        if( entities[dim].empty() ) continue;
        num_elements.resize( entities[dim].size() );
        int i = 0;
        for( Range::iterator rit = entities[dim].begin(); rit != entities[dim].end(); ++rit )
            num_elements[i++] = total_elems[dim]++;
 
        result = mbImpl->tag_set_data( gid_tag, entities[dim], &num_elements[0] );MB_CHK_SET_ERR( result, "Failed to set global id tag in assign_global_ids" );
    }
 
    if( owned_only ) return MB_SUCCESS;
 
    // Exchange tags
    for( int dim = 1; dim < 4; dim++ )
        entities[0].merge( entities[dim] );
 
    return exchange_tags( gid_tag, entities[0] );
}
 
int ParallelComm::get_buffers( int to_proc, bool* is_new )
{
    int ind                                   = -1;
    std::vector< unsigned int >::iterator vit = std::find( buffProcs.begin(), buffProcs.end(), to_proc );
    if( vit == buffProcs.end() )
    {
        assert( "shouldn't need buffer to myself" && to_proc != (int)procConfig.proc_rank() );
        ind = buffProcs.size();
        buffProcs.push_back( (unsigned int)to_proc );
        localOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
        remoteOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
        if( is_new ) *is_new = true;
    }
    else
    {
        ind = vit - buffProcs.begin();
        if( is_new ) *is_new = false;
    }
    assert( ind < MAX_SHARING_PROCS );
    return ind;
}
 
ErrorCode ParallelComm::broadcast_entities( const int from_proc,
                                            Range& entities,
                                            const bool adjacencies,
                                            const bool tags )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
 
    ErrorCode result = MB_SUCCESS;
    int success;
    int buff_size;
 
    Buffer buff( INITIAL_BUFF_SIZE );
    buff.reset_ptr( sizeof( int ) );
    if( (int)procConfig.proc_rank() == from_proc )
    {
        result = add_verts( entities );MB_CHK_SET_ERR( result, "Failed to add adj vertices" );
 
        buff.reset_ptr( sizeof( int ) );
        result = pack_buffer( entities, adjacencies, tags, false, -1, &buff );MB_CHK_SET_ERR( result, "Failed to compute buffer size in broadcast_entities" );
        buff.set_stored_size();
        buff_size = buff.buff_ptr - buff.mem_ptr;
    }
 
    success = MPI_Bcast( &buff_size, 1, MPI_INT, from_proc, procConfig.proc_comm() );
    if( MPI_SUCCESS != success )
    {
        MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
    }
 
    if( !buff_size )  // No data
        return MB_SUCCESS;
 
    if( (int)procConfig.proc_rank() != from_proc ) buff.reserve( buff_size );
 
    size_t offset = 0;
    while( buff_size )
    {
        int sz  = std::min( buff_size, MAX_BCAST_SIZE );
        success = MPI_Bcast( buff.mem_ptr + offset, sz, MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer failed" );
        }
 
        offset += sz;
        buff_size -= sz;
    }
 
    if( (int)procConfig.proc_rank() != from_proc )
    {
        std::vector< std::vector< EntityHandle > > dum1a, dum1b;
        std::vector< std::vector< int > > dum1p;
        std::vector< EntityHandle > dum2, dum4;
        std::vector< unsigned int > dum3;
        buff.reset_ptr( sizeof( int ) );
        result = unpack_buffer( buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );MB_CHK_SET_ERR( result, "Failed to unpack buffer in broadcast_entities" );
        std::copy( dum4.begin(), dum4.end(), range_inserter( entities ) );
    }
 
    return MB_SUCCESS;
#endif
}
 
ErrorCode ParallelComm::scatter_entities( const int from_proc,
                                          std::vector< Range >& entities,
                                          const bool adjacencies,
                                          const bool tags )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
    ErrorCode result = MB_SUCCESS;
    int i, success, buff_size, prev_size;
    int nProcs         = (int)procConfig.proc_size();
    int* sendCounts    = new int[nProcs];
    int* displacements = new int[nProcs];
    sendCounts[0]      = sizeof( int );
    displacements[0]   = 0;
    Buffer buff( INITIAL_BUFF_SIZE );
    buff.reset_ptr( sizeof( int ) );
    buff.set_stored_size();
    unsigned int my_proc = procConfig.proc_rank();
 
    // Get buffer size array for each remote processor
    if( my_proc == (unsigned int)from_proc )
    {
        for( i = 1; i < nProcs; i++ )
        {
            prev_size = buff.buff_ptr - buff.mem_ptr;
            buff.reset_ptr( prev_size + sizeof( int ) );
            result = add_verts( entities[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
 
            result = pack_buffer( entities[i], adjacencies, tags, false, -1, &buff );
            if( MB_SUCCESS != result )
            {
                delete[] sendCounts;
                delete[] displacements;
                MB_SET_ERR( result, "Failed to pack buffer in scatter_entities" );
            }
 
            buff_size                               = buff.buff_ptr - buff.mem_ptr - prev_size;
            *( (int*)( buff.mem_ptr + prev_size ) ) = buff_size;
            sendCounts[i]                           = buff_size;
        }
    }
 
    // Broadcast buffer size array
    success = MPI_Bcast( sendCounts, nProcs, MPI_INT, from_proc, procConfig.proc_comm() );
    if( MPI_SUCCESS != success )
    {
        delete[] sendCounts;
        delete[] displacements;
        MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
    }
 
    for( i = 1; i < nProcs; i++ )
    {
        displacements[i] = displacements[i - 1] + sendCounts[i - 1];
    }
 
    Buffer rec_buff;
    rec_buff.reserve( sendCounts[my_proc] );
 
    // Scatter actual geometry
    success = MPI_Scatterv( buff.mem_ptr, sendCounts, displacements, MPI_UNSIGNED_CHAR, rec_buff.mem_ptr,
                            sendCounts[my_proc], MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
 
    if( MPI_SUCCESS != success )
    {
        delete[] sendCounts;
        delete[] displacements;
        MB_SET_ERR( MB_FAILURE, "MPI_Scatterv of buffer failed" );
    }
 
    // Unpack in remote processors
    if( my_proc != (unsigned int)from_proc )
    {
        std::vector< std::vector< EntityHandle > > dum1a, dum1b;
        std::vector< std::vector< int > > dum1p;
        std::vector< EntityHandle > dum2, dum4;
        std::vector< unsigned int > dum3;
        rec_buff.reset_ptr( sizeof( int ) );
        result = unpack_buffer( rec_buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );
        if( MB_SUCCESS != result )
        {
            delete[] sendCounts;
            delete[] displacements;
            MB_SET_ERR( result, "Failed to unpack buffer in scatter_entities" );
        }
 
        std::copy( dum4.begin(), dum4.end(), range_inserter( entities[my_proc] ) );
    }
 
    delete[] sendCounts;
    delete[] displacements;
 
    return MB_SUCCESS;
#endif
}
 
ErrorCode ParallelComm::send_entities( const int to_proc,
                                       Range& orig_ents,
                                       const bool adjs,
                                       const bool tags,
                                       const bool store_remote_handles,
                                       const bool is_iface,
                                       Range& /*final_ents*/,
                                       int& incoming1,
                                       int& incoming2,
                                       TupleList& entprocs,
                                       std::vector< MPI_Request >& recv_remoteh_reqs,
                                       bool /*wait_all*/ )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
    // Pack entities to local buffer
    int ind = get_buffers( to_proc );
    localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
 
    // Add vertices
    ErrorCode result = add_verts( orig_ents );MB_CHK_SET_ERR( result, "Failed to add verts in send_entities" );
 
    // Filter out entities already shared with destination
    Range tmp_range;
    result = filter_pstatus( orig_ents, PSTATUS_SHARED, PSTATUS_AND, to_proc, &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
    if( !tmp_range.empty() )
    {
        orig_ents = subtract( orig_ents, tmp_range );
    }
 
    result = pack_buffer( orig_ents, adjs, tags, store_remote_handles, to_proc, localOwnedBuffs[ind], &entprocs );MB_CHK_SET_ERR( result, "Failed to pack buffer in send_entities" );
 
    // Send buffer
    result = send_buffer( to_proc, localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], recvReqs[2 * ind + 1],
                          (int*)( remoteOwnedBuffs[ind]->mem_ptr ),
                          //&ackbuff,
                          incoming1, MB_MESG_REMOTEH_SIZE,
                          ( !is_iface && store_remote_handles ? localOwnedBuffs[ind] : NULL ),
                          &recv_remoteh_reqs[2 * ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to send buffer" );
 
    return MB_SUCCESS;
#endif
}
 
ErrorCode ParallelComm::send_entities( std::vector< unsigned int >& send_procs,
                                       std::vector< Range* >& send_ents,
                                       int& incoming1,
                                       int& incoming2,
                                       const bool store_remote_handles )
{
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_entities." );
    }
#endif
    myDebug->tprintf( 1, "Entering send_entities\n" );
    if( myDebug->get_verbosity() == 4 )
    {
        msgs.clear();
        msgs.reserve( MAX_SHARING_PROCS );
    }
 
    unsigned int i;
    int ind;
    ErrorCode result = MB_SUCCESS;
 
    // Set buffProcs with communicating procs
    unsigned int n_proc = send_procs.size();
    for( i = 0; i < n_proc; i++ )
    {
        ind    = get_buffers( send_procs[i] );
        result = add_verts( *send_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
 
        // Filter out entities already shared with destination
        Range tmp_range;
        result = filter_pstatus( *send_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
        if( !tmp_range.empty() )
        {
            *send_ents[i] = subtract( *send_ents[i], tmp_range );
        }
    }
 
    //===========================================
    // Get entities to be sent to neighbors
    // Need to get procs each entity is sent to
    //===========================================
    Range allsent, tmp_range;
    int npairs = 0;
    TupleList entprocs;
    for( i = 0; i < n_proc; i++ )
    {
        int n_ents = send_ents[i]->size();
        if( n_ents > 0 )
        {
            npairs += n_ents;  // Get the total # of proc/handle pairs
            allsent.merge( *send_ents[i] );
        }
    }
 
    // Allocate a TupleList of that size
    entprocs.initialize( 1, 0, 1, 0, npairs );
    entprocs.enableWriteAccess();
 
    // Put the proc/handle pairs in the list
    for( i = 0; i < n_proc; i++ )
    {
        for( Range::iterator rit = send_ents[i]->begin(); rit != send_ents[i]->end(); ++rit )
        {
            entprocs.vi_wr[entprocs.get_n()]  = send_procs[i];
            entprocs.vul_wr[entprocs.get_n()] = *rit;
            entprocs.inc_n();
        }
    }
 
    // Sort by handle
    moab::TupleList::buffer sort_buffer;
    sort_buffer.buffer_init( npairs );
    entprocs.sort( 1, &sort_buffer );
    entprocs.disableWriteAccess();
    sort_buffer.reset();
 
    myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
                      (unsigned long)allsent.size() );
 
    //===========================================
    // Pack and send ents from this proc to others
    //===========================================
    for( i = 0; i < n_proc; i++ )
    {
        if( send_ents[i]->size() > 0 )
        {
            ind = get_buffers( send_procs[i] );
            myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", send_ents[i]->compactness(),
                              (unsigned long)send_ents[i]->size() );
            // Reserve space on front for size and for initial buff size
            localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
            result = pack_buffer( *send_ents[i], false, true, store_remote_handles, buffProcs[ind],
                                  localOwnedBuffs[ind], &entprocs, &allsent );
 
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *localOwnedBuffs[ind] );
            }
 
            // Send the buffer (size stored in front in send_buffer)
            result = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind],
                                  recvReqs[2 * ind + 1], &ackbuff, incoming1, MB_MESG_REMOTEH_SIZE,
                                  ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recvRemotehReqs[2 * ind],
                                  &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost send" );
        }
    }
    entprocs.reset();
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_entities." );
    }
#endif
 
    return MB_SUCCESS;
}
 
/////////////////////////////////////////////////////////////////////////////////
// Send and Receive routines for a sequence of entities: use case UMR
/////////////////////////////////////////////////////////////////////////////////
void print_buff( unsigned char* ch, int size )
{
    for( int i = 0; i < size; i++ )
        std::cout << ch[i];
    std::cout << "\n";
}
ErrorCode ParallelComm::send_recv_entities( std::vector< int >& send_procs,
                                            std::vector< std::vector< int > >& msgsizes,
                                            std::vector< std::vector< EntityHandle > >& senddata,
                                            std::vector< std::vector< EntityHandle > >& recvdata )
{
#ifdef USE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_recv_entities." );
    }
#endif
    myDebug->tprintf( 1, "Entering send_recv_entities\n" );
    if( myDebug->get_verbosity() == 4 )
    {
        msgs.clear();
        msgs.reserve( MAX_SHARING_PROCS );
    }
 
    // unsigned int i;
    int i, ind, success;
    ErrorCode error = MB_SUCCESS;
 
    //===========================================
    // Pack and send ents from this proc to others
    //===========================================
 
    // std::cout<<"resetting all buffers"<<std::endl;
 
    reset_all_buffers();
    sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    int ack_buff;
    int incoming = 0;
 
    std::vector< unsigned int >::iterator sit;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        incoming++;
        PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_ENTS_SIZE, incoming );
 
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
                             MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in send_recv_entities" );
        }
    }
 
    //  std::set<unsigned int>::iterator it;
    for( i = 0; i < (int)send_procs.size(); i++ )
    {
        // Get index of the shared processor in the local buffer
        ind = get_buffers( send_procs[i] );
        localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
 
        int buff_size = msgsizes[i].size() * sizeof( int ) + senddata[i].size() * sizeof( EntityHandle );
        localOwnedBuffs[ind]->check_space( buff_size );
 
        // Pack entities
        std::vector< int > msg;
        msg.insert( msg.end(), msgsizes[i].begin(), msgsizes[i].end() );
        PACK_INTS( localOwnedBuffs[ind]->buff_ptr, &msg[0], msg.size() );
 
        std::vector< EntityHandle > entities;
        entities.insert( entities.end(), senddata[i].begin(), senddata[i].end() );
        PACK_EH( localOwnedBuffs[ind]->buff_ptr, &entities[0], entities.size() );
        localOwnedBuffs[ind]->set_stored_size();
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *localOwnedBuffs[ind] );
        }
 
        // Send the buffer (size stored in front in send_buffer)
        error = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind],
                             recv_ent_reqs[3 * ind + 2], &ack_buff, incoming );MB_CHK_SET_ERR( error, "Failed to Isend in send_recv_entities" );
    }
 
    //===========================================
    // Receive and unpack ents from received data
    //===========================================
 
    while( incoming )
    {
 
        MPI_Status status;
        int index_in_recv_requests;
 
        PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &index_in_recv_requests, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in send_recv_entities" );
        }
 
        // Processor index in the list is divided by 3
        ind = index_in_recv_requests / 3;
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming--;
 
        bool done = false;
 
        error = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind],
                             recv_ent_reqs[3 * ind + 1],  // This is for receiving the second message
                             recv_ent_reqs[3 * ind + 2],  // This would be for ack, but it is not
                                                          // used; consider removing it
                             incoming, localOwnedBuffs[ind],
                             sendReqs[3 * ind + 1],  // Send request for sending the second message
                             sendReqs[3 * ind + 2],  // This is for sending the ack
                             done );MB_CHK_SET_ERR( error, "Failed to resize recv buffer" );
 
        if( done )
        {
            remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
 
            int from_proc = status.MPI_SOURCE;
            int idx       = std::find( send_procs.begin(), send_procs.end(), from_proc ) - send_procs.begin();
 
            int msg = msgsizes[idx].size();
            std::vector< int > recvmsg( msg );
            int ndata = senddata[idx].size();
            std::vector< EntityHandle > dum_vec( ndata );
 
            UNPACK_INTS( remoteOwnedBuffs[ind]->buff_ptr, &recvmsg[0], msg );
            UNPACK_EH( remoteOwnedBuffs[ind]->buff_ptr, &dum_vec[0], ndata );
 
            recvdata[idx].insert( recvdata[idx].end(), dum_vec.begin(), dum_vec.end() );
        }
    }
 
#ifdef USE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_recv_entities." );
    }
#endif
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::update_remote_data( EntityHandle entity,
                                            std::vector< int >& procs,
                                            std::vector< EntityHandle >& handles )
{
    ErrorCode error;
    unsigned char pstatus = PSTATUS_INTERFACE;
 
    int procmin = *std::min_element( procs.begin(), procs.end() );
 
    if( (int)rank() > procmin )
        pstatus |= PSTATUS_NOT_OWNED;
    else
        procmin = rank();
 
    // DBG
    // std::cout<<"entity = "<<entity<<std::endl;
    // for (int j=0; j<procs.size(); j++)
    // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl;
    // DBG
 
    if( (int)procs.size() > 1 )
    {
        procs.push_back( rank() );
        handles.push_back( entity );
 
        int idx = std::find( procs.begin(), procs.end(), procmin ) - procs.begin();
 
        std::iter_swap( procs.begin(), procs.begin() + idx );
        std::iter_swap( handles.begin(), handles.begin() + idx );
 
        // DBG
        //  std::cout<<"entity = "<<entity<<std::endl;
        // for (int j=0; j<procs.size(); j++)
        // std::cout<<"procs["<<j<<"] = "<<procs[j]<<", handles["<<j<<"] = "<<handles[j]<<std::endl;
        // DBG
    }
 
    // if ((entity == 10388) && (rank()==1))
    //    std::cout<<"Here"<<std::endl;
 
    error = update_remote_data( entity, &procs[0], &handles[0], procs.size(), pstatus );MB_CHK_ERR( error );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc )
{
    ErrorCode error;
    std::vector< EntityHandle > newents;
    error = get_remote_handles( true, local_vec, rem_vec, num_ents, to_proc, newents );MB_CHK_ERR( error );
 
    return MB_SUCCESS;
}
 
//////////////////////////////////////////////////////////////////
 
ErrorCode ParallelComm::recv_entities( const int from_proc,
                                       const bool store_remote_handles,
                                       const bool is_iface,
                                       Range& final_ents,
                                       int& incoming1,
                                       int& incoming2,
                                       std::vector< std::vector< EntityHandle > >& L1hloc,
                                       std::vector< std::vector< EntityHandle > >& L1hrem,
                                       std::vector< std::vector< int > >& L1p,
                                       std::vector< EntityHandle >& L2hloc,
                                       std::vector< EntityHandle >& L2hrem,
                                       std::vector< unsigned int >& L2p,
                                       std::vector< MPI_Request >& recv_remoteh_reqs,
                                       bool /*wait_all*/ )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
    // Non-blocking receive for the first message (having size info)
    int ind1 = get_buffers( from_proc );
    incoming1++;
    PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE,
                       MB_MESG_ENTS_SIZE, incoming1 );
    int success = MPI_Irecv( remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc,
                             MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind1] );
    if( success != MPI_SUCCESS )
    {
        MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
    }
 
    // Receive messages in while loop
    return recv_messages( from_proc, store_remote_handles, is_iface, final_ents, incoming1, incoming2, L1hloc, L1hrem,
                          L1p, L2hloc, L2hrem, L2p, recv_remoteh_reqs );
#endif
}
 
ErrorCode ParallelComm::recv_entities( std::set< unsigned int >& recv_procs,
                                       int incoming1,
                                       int incoming2,
                                       const bool store_remote_handles,
                                       const bool migrate )
{
    //===========================================
    // Receive/unpack new entities
    //===========================================
    // Number of incoming messages is the number of procs we communicate with
    int success, ind, i;
    ErrorCode result;
    MPI_Status status;
    std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
    std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
    std::vector< std::vector< int > > L1p( buffProcs.size() );
    std::vector< EntityHandle > L2hloc, L2hrem;
    std::vector< unsigned int > L2p;
    std::vector< EntityHandle > new_ents;
 
    while( incoming1 )
    {
        // Wait for all recvs of ents before proceeding to sending remote handles,
        // b/c some procs may have sent to a 3rd proc ents owned by me;
        PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
 
        success = MPI_Waitany( 2 * buffProcs.size(), &recvReqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
        }
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming1--;
        bool done = false;
 
        // In case ind is for ack, we need index of one before it
        unsigned int base_ind = 2 * ( ind / 2 );
        result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 2], recvReqs[ind], recvReqs[ind + 1],
                              incoming1, localOwnedBuffs[ind / 2], sendReqs[base_ind], sendReqs[base_ind + 1], done,
                              ( store_remote_handles ? localOwnedBuffs[ind / 2] : NULL ), MB_MESG_REMOTEH_SIZE,
                              &recvRemotehReqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
 
        if( done )
        {
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 2] );
            }
 
            // Message completely received - process buffer that was sent
            remoteOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
            result = unpack_buffer( remoteOwnedBuffs[ind / 2]->buff_ptr, store_remote_handles, buffProcs[ind / 2],
                                    ind / 2, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true );
            if( MB_SUCCESS != result )
            {
                std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
                print_buffer( remoteOwnedBuffs[ind / 2]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 2], false );
                return result;
            }
 
            if( recvReqs.size() != 2 * buffProcs.size() )
            {
                // Post irecv's for remote handles from new proc
                recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
                for( i = recvReqs.size(); i < (int)( 2 * buffProcs.size() ); i += 2 )
                {
                    localOwnedBuffs[i / 2]->reset_buffer();
                    incoming2++;
                    PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 2], localOwnedBuffs[i / 2]->mem_ptr,
                                       INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
                    success = MPI_Irecv( localOwnedBuffs[i / 2]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
                                         buffProcs[i / 2], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
                                         &recvRemotehReqs[i] );
                    if( success != MPI_SUCCESS )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
                    }
                }
                recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
                sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
            }
        }
    }
 
    // Assign and remove newly created elements from/to receive processor
    result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" );
    if( migrate )
    {
        // result = remove_entities_part(allsent, procConfig.proc_rank());MB_CHK_SET_ERR(ressult,
        // "Failed to remove entities to part");
    }
 
    // Add requests for any new addl procs
    if( recvReqs.size() != 2 * buffProcs.size() )
    {
        // Shouldn't get here...
        MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" );
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending recv entities." );
    }
#endif
 
    //===========================================
    // Send local handles for new entity to owner
    //===========================================
    std::set< unsigned int >::iterator it  = recv_procs.begin();
    std::set< unsigned int >::iterator eit = recv_procs.end();
    for( ; it != eit; ++it )
    {
        ind = get_buffers( *it );
        // Reserve space on front for size and for initial buff size
        remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
 
        result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
        remoteOwnedBuffs[ind]->set_stored_size();
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *remoteOwnedBuffs[ind] );
        }
        result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[2 * ind],
                              recvRemotehReqs[2 * ind + 1], &ackbuff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
    }
 
    //===========================================
    // Process remote handles of my ghosteds
    //===========================================
    while( incoming2 )
    {
        PRINT_DEBUG_WAITANY( recvRemotehReqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 2 * buffProcs.size(), &recvRemotehReqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
        }
 
        // OK, received something; decrement incoming counter
        incoming2--;
 
        PRINT_DEBUG_RECD( status );
        bool done             = false;
        unsigned int base_ind = 2 * ( ind / 2 );
        result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 2], recvRemotehReqs[ind],
                              recvRemotehReqs[ind + 1], incoming2, remoteOwnedBuffs[ind / 2], sendReqs[base_ind],
                              sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
        if( done )
        {
            // Incoming remote handles
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *localOwnedBuffs[ind] );
            }
 
            localOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
            result =
                unpack_remote_handles( buffProcs[ind / 2], localOwnedBuffs[ind / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
        }
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
        MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending recv entities (still doing checks)." );
    }
#endif
    myDebug->tprintf( 1, "Exiting recv_entities.\n" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::recv_messages( const int from_proc,
                                       const bool store_remote_handles,
                                       const bool is_iface,
                                       Range& final_ents,
                                       int& incoming1,
                                       int& incoming2,
                                       std::vector< std::vector< EntityHandle > >& L1hloc,
                                       std::vector< std::vector< EntityHandle > >& L1hrem,
                                       std::vector< std::vector< int > >& L1p,
                                       std::vector< EntityHandle >& L2hloc,
                                       std::vector< EntityHandle >& L2hrem,
                                       std::vector< unsigned int >& L2p,
                                       std::vector< MPI_Request >& recv_remoteh_reqs )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
    MPI_Status status;
    ErrorCode result;
    int ind1 = get_buffers( from_proc );
    int success, ind2;
    std::vector< EntityHandle > new_ents;
 
    // Wait and receive messages
    while( incoming1 )
    {
        PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 2, &recvReqs[2 * ind1], &ind2, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_messages" );
        }
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming1--;
        bool done = false;
 
        // In case ind is for ack, we need index of one before it
        ind2 += 2 * ind1;
        unsigned int base_ind = 2 * ( ind2 / 2 );
 
        result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind2 / 2],
                              // recvbuff,
                              recvReqs[ind2], recvReqs[ind2 + 1], incoming1, localOwnedBuffs[ind2 / 2],
                              sendReqs[base_ind], sendReqs[base_ind + 1], done,
                              ( !is_iface && store_remote_handles ? localOwnedBuffs[ind2 / 2] : NULL ),
                              MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
 
        if( done )
        {
            // If it is done, unpack buffer
            remoteOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
            result = unpack_buffer( remoteOwnedBuffs[ind2 / 2]->buff_ptr, store_remote_handles, from_proc, ind2 / 2,
                                    L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );MB_CHK_SET_ERR( result, "Failed to unpack buffer in recev_messages" );
 
            std::copy( new_ents.begin(), new_ents.end(), range_inserter( final_ents ) );
 
            // Send local handles for new elements to owner
            // Reserve space on front for size and for initial buff size
            remoteOwnedBuffs[ind2 / 2]->reset_buffer( sizeof( int ) );
 
            result = pack_remote_handles( L1hloc[ind2 / 2], L1hrem[ind2 / 2], L1p[ind2 / 2], from_proc,
                                          remoteOwnedBuffs[ind2 / 2] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
            remoteOwnedBuffs[ind2 / 2]->set_stored_size();
 
            result = send_buffer( buffProcs[ind2 / 2], remoteOwnedBuffs[ind2 / 2], MB_MESG_REMOTEH_SIZE, sendReqs[ind2],
                                  recv_remoteh_reqs[ind2 + 1], (int*)( localOwnedBuffs[ind2 / 2]->mem_ptr ),
                                  //&ackbuff,
                                  incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
        }
    }
 
    return MB_SUCCESS;
#endif
}
 
ErrorCode ParallelComm::recv_remote_handle_messages( const int from_proc,
                                                     int& incoming2,
                                                     std::vector< EntityHandle >& L2hloc,
                                                     std::vector< EntityHandle >& L2hrem,
                                                     std::vector< unsigned int >& L2p,
                                                     std::vector< MPI_Request >& recv_remoteh_reqs )
{
#ifndef MOAB_HAVE_MPI
    return MB_FAILURE;
#else
    MPI_Status status;
    ErrorCode result;
    int ind1 = get_buffers( from_proc );
    int success, ind2;
 
    while( incoming2 )
    {
        PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 2, &recv_remoteh_reqs[2 * ind1], &ind2, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_remote_handle_messages" );
        }
 
        // OK, received something; decrement incoming counter
        incoming2--;
 
        PRINT_DEBUG_RECD( status );
 
        bool done = false;
        ind2 += 2 * ind1;
        unsigned int base_ind = 2 * ( ind2 / 2 );
        result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind2 / 2], recv_remoteh_reqs[ind2],
                              recv_remoteh_reqs[ind2 + 1], incoming2, remoteOwnedBuffs[ind2 / 2], sendReqs[base_ind],
                              sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
        if( done )
        {
            // Incoming remote handles
            localOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
            result =
                unpack_remote_handles( buffProcs[ind2 / 2], localOwnedBuffs[ind2 / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
        }
    }
 
    return MB_SUCCESS;
#endif
}
 
ErrorCode ParallelComm::pack_buffer( Range& orig_ents,
                                     const bool /*adjacencies*/,
                                     const bool tags,
                                     const bool store_remote_handles,
                                     const int to_proc,
                                     Buffer* buff,
                                     TupleList* entprocs,
                                     Range* allsent )
{
    // Pack the buffer with the entity ranges, adjacencies, and tags sections
    //
    // Note: new entities used in subsequent connectivity lists, sets, or tags,
    // are referred to as (MBMAXTYPE + index), where index is into vector
    // of new entities, 0-based
    ErrorCode result;
 
    Range set_range;
    std::vector< Tag > all_tags;
    std::vector< Range > tag_ranges;
 
    Range::const_iterator rit;
 
    // Entities
    result = pack_entities( orig_ents, buff, store_remote_handles, to_proc, false, entprocs, allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
 
    // Sets
    result = pack_sets( orig_ents, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing sets (count) failed" );
 
    // Tags
    Range final_ents;
    if( tags )
    {
        result = get_tag_send_list( orig_ents, all_tags, tag_ranges );MB_CHK_SET_ERR( result, "Failed to get tagged entities" );
        result = pack_tags( orig_ents, all_tags, all_tags, tag_ranges, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing tags (count) failed" );
    }
    else
    {  // Set tag size to 0
        buff->check_space( sizeof( int ) );
        PACK_INT( buff->buff_ptr, 0 );
        buff->set_stored_size();
    }
 
    return result;
}
 
ErrorCode ParallelComm::unpack_buffer( unsigned char* buff_ptr,
                                       const bool store_remote_handles,
                                       const int from_proc,
                                       const int ind,
                                       std::vector< std::vector< EntityHandle > >& L1hloc,
                                       std::vector< std::vector< EntityHandle > >& L1hrem,
                                       std::vector< std::vector< int > >& L1p,
                                       std::vector< EntityHandle >& L2hloc,
                                       std::vector< EntityHandle >& L2hrem,
                                       std::vector< unsigned int >& L2p,
                                       std::vector< EntityHandle >& new_ents,
                                       const bool created_iface )
{
    unsigned char* tmp_buff = buff_ptr;
    ErrorCode result;
    result = unpack_entities( buff_ptr, store_remote_handles, ind, false, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p,
                              new_ents, created_iface );MB_CHK_SET_ERR( result, "Unpacking entities failed" );
    if( myDebug->get_verbosity() == 3 )
    {
        myDebug->tprintf( 4, "unpack_entities buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
        tmp_buff = buff_ptr;
    }
    result = unpack_sets( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking sets failed" );
    if( myDebug->get_verbosity() == 3 )
    {
        myDebug->tprintf( 4, "unpack_sets buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
        tmp_buff = buff_ptr;
    }
    result = unpack_tags( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking tags failed" );
    if( myDebug->get_verbosity() == 3 )
    {
        myDebug->tprintf( 4, "unpack_tags buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
        // tmp_buff = buff_ptr;
    }
 
    if( myDebug->get_verbosity() == 3 ) myDebug->print( 4, "\n" );
 
    return MB_SUCCESS;
}
 
int ParallelComm::estimate_ents_buffer_size( Range& entities, const bool store_remote_handles )
{
    int buff_size = 0;
    std::vector< EntityHandle > dum_connect_vec;
    const EntityHandle* connect;
    int num_connect;
 
    int num_verts = entities.num_of_type( MBVERTEX );
    // # verts + coords + handles
    buff_size += 2 * sizeof( int ) + 3 * sizeof( double ) * num_verts;
    if( store_remote_handles ) buff_size += sizeof( EntityHandle ) * num_verts;
 
    // Do a rough count by looking at first entity of each type
    for( EntityType t = MBEDGE; t < MBENTITYSET; t++ )
    {
        const Range::iterator rit = entities.lower_bound( t );
        if( TYPE_FROM_HANDLE( *rit ) != t ) continue;
 
        ErrorCode result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect_vec );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get connectivity to estimate buffer size", -1 );
 
        // Number, type, nodes per entity
        buff_size += 3 * sizeof( int );
        int num_ents = entities.num_of_type( t );
        // Connectivity, handle for each ent
        buff_size += ( num_connect + 1 ) * sizeof( EntityHandle ) * num_ents;
    }
 
    // Extra entity type at end, passed as int
    buff_size += sizeof( int );
 
    return buff_size;
}
 
int ParallelComm::estimate_sets_buffer_size( Range& entities, const bool /*store_remote_handles*/ )
{
    // Number of sets
    int buff_size = sizeof( int );
 
    // Do a rough count by looking at first entity of each type
    Range::iterator rit = entities.lower_bound( MBENTITYSET );
    ErrorCode result;
 
    for( ; rit != entities.end(); ++rit )
    {
        unsigned int options;
        result = mbImpl->get_meshset_options( *rit, options );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get meshset options", -1 );
 
        buff_size += sizeof( int );
 
        Range set_range;
        if( options & MESHSET_SET )
        {
            // Range-based set; count the subranges
            result = mbImpl->get_entities_by_handle( *rit, set_range );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get set entities", -1 );
 
            // Set range
            buff_size += RANGE_SIZE( set_range );
        }
        else if( options & MESHSET_ORDERED )
        {
            // Just get the number of entities in the set
            int num_ents;
            result = mbImpl->get_number_entities_by_handle( *rit, num_ents );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get number entities in ordered set", -1 );
 
            // Set vec
            buff_size += sizeof( EntityHandle ) * num_ents + sizeof( int );
        }
 
        // Get numbers of parents/children
        int num_par, num_ch;
        result = mbImpl->num_child_meshsets( *rit, &num_ch );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num children", -1 );
        result = mbImpl->num_parent_meshsets( *rit, &num_par );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num parents", -1 );
 
        buff_size += ( num_ch + num_par ) * sizeof( EntityHandle ) + 2 * sizeof( int );
    }
 
    return buff_size;
}
 
ErrorCode ParallelComm::pack_entities( Range& entities,
                                       Buffer* buff,
                                       const bool store_remote_handles,
                                       const int to_proc,
                                       const bool /*is_iface*/,
                                       TupleList* entprocs,
                                       Range* /*allsent*/ )
{
    // Packed information:
    // 1. # entities = E
    // 2. for e in E
    //   a. # procs sharing e, incl. sender and receiver = P
    //   b. for p in P (procs sharing e)
    //   c. for p in P (handle for e on p) (Note1)
    // 3. vertex/entity info
 
    // Get an estimate of the buffer size & pre-allocate buffer size
    int buff_size = estimate_ents_buffer_size( entities, store_remote_handles );
    if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate ents buffer size" );
    buff->check_space( buff_size );
    myDebug->tprintf( 3, "estimate buffer size for %d entities: %d \n", (int)entities.size(), buff_size );
 
    unsigned int num_ents;
    ErrorCode result;
 
    std::vector< EntityHandle > entities_vec( entities.size() );
    std::copy( entities.begin(), entities.end(), entities_vec.begin() );
 
    // First pack procs/handles sharing this ent, not including this dest but including
    // others (with zero handles)
    if( store_remote_handles )
    {
        // Buff space is at least proc + handle for each entity; use avg of 4 other procs
        // to estimate buff size, but check later
        buff->check_space( sizeof( int ) + ( 5 * sizeof( int ) + sizeof( EntityHandle ) ) * entities.size() );
 
        // 1. # entities = E
        PACK_INT( buff->buff_ptr, entities.size() );
 
        Range::iterator rit;
 
        // Pre-fetch sharedp and pstatus
        std::vector< int > sharedp_vals( entities.size() );
        result = mbImpl->tag_get_data( sharedp_tag(), entities, &sharedp_vals[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
        std::vector< char > pstatus_vals( entities.size() );
        result = mbImpl->tag_get_data( pstatus_tag(), entities, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
 
        unsigned int i;
        int tmp_procs[MAX_SHARING_PROCS];
        EntityHandle tmp_handles[MAX_SHARING_PROCS];
        std::set< unsigned int > dumprocs;
 
        // 2. for e in E
        for( rit = entities.begin(), i = 0; rit != entities.end(); ++rit, i++ )
        {
            unsigned int ind =
                std::lower_bound( entprocs->vul_rd, entprocs->vul_rd + entprocs->get_n(), *rit ) - entprocs->vul_rd;
            assert( ind < entprocs->get_n() );
 
            while( ind < entprocs->get_n() && entprocs->vul_rd[ind] == *rit )
                dumprocs.insert( entprocs->vi_rd[ind++] );
 
            result = build_sharedhps_list( *rit, pstatus_vals[i], sharedp_vals[i], dumprocs, num_ents, tmp_procs,
                                           tmp_handles );MB_CHK_SET_ERR( result, "Failed to build sharedhps" );
 
            dumprocs.clear();
 
            // Now pack them
            buff->check_space( ( num_ents + 1 ) * sizeof( int ) + num_ents * sizeof( EntityHandle ) );
            PACK_INT( buff->buff_ptr, num_ents );
            PACK_INTS( buff->buff_ptr, tmp_procs, num_ents );
            PACK_EH( buff->buff_ptr, tmp_handles, num_ents );
 
#ifndef NDEBUG
            // Check for duplicates in proc list
            unsigned int dp = 0;
            for( ; dp < MAX_SHARING_PROCS && -1 != tmp_procs[dp]; dp++ )
                dumprocs.insert( tmp_procs[dp] );
            assert( dumprocs.size() == dp );
            dumprocs.clear();
#endif
        }
    }
 
    // Pack vertices
    Range these_ents = entities.subset_by_type( MBVERTEX );
    num_ents         = these_ents.size();
 
    if( num_ents )
    {
        buff_size = 2 * sizeof( int ) + 3 * num_ents * sizeof( double );
        buff->check_space( buff_size );
 
        // Type, # ents
        PACK_INT( buff->buff_ptr, ( (int)MBVERTEX ) );
        PACK_INT( buff->buff_ptr, ( (int)num_ents ) );
 
        std::vector< double > tmp_coords( 3 * num_ents );
        result = mbImpl->get_coords( these_ents, &tmp_coords[0] );MB_CHK_SET_ERR( result, "Failed to get vertex coordinates" );
        PACK_DBLS( buff->buff_ptr, &tmp_coords[0], 3 * num_ents );
 
        myDebug->tprintf( 4, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
                          CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
    }
 
    // Now entities; go through range, packing by type and equal # verts per element
    Range::iterator start_rit = entities.find( *these_ents.rbegin() );
    ++start_rit;
    int last_nodes       = -1;
    EntityType last_type = MBMAXTYPE;
    these_ents.clear();
    Range::iterator end_rit = start_rit;
    EntitySequence* seq;
    ElementSequence* eseq;
 
    while( start_rit != entities.end() || !these_ents.empty() )
    {
        // Cases:
        // A: !end, last_type == MBMAXTYPE, seq: save contig sequence in these_ents
        // B: !end, last type & nodes same, seq: save contig sequence in these_ents
        // C: !end, last type & nodes different: pack these_ents, then save contig sequence in
        // these_ents D: end: pack these_ents
 
        // Find the sequence holding current start entity, if we're not at end
        eseq = NULL;
        if( start_rit != entities.end() )
        {
            result = sequenceManager->find( *start_rit, seq );MB_CHK_SET_ERR( result, "Failed to find entity sequence" );
            if( NULL == seq ) return MB_FAILURE;
            eseq = dynamic_cast< ElementSequence* >( seq );
        }
 
        // Pack the last batch if at end or next one is different
        if( !these_ents.empty() &&
            ( !eseq || eseq->type() != last_type || last_nodes != (int)eseq->nodes_per_element() ) )
        {
            result = pack_entity_seq( last_nodes, store_remote_handles, to_proc, these_ents, entities_vec, buff );MB_CHK_SET_ERR( result, "Failed to pack entities from a sequence" );
            these_ents.clear();
        }
 
        if( eseq )
        {
            // Continuation of current range, just save these entities
            // Get position in entities list one past end of this sequence
            end_rit = entities.lower_bound( start_rit, entities.end(), eseq->end_handle() + 1 );
 
            // Put these entities in the range
            std::copy( start_rit, end_rit, range_inserter( these_ents ) );
 
            last_type  = eseq->type();
            last_nodes = eseq->nodes_per_element();
        }
        else if( start_rit != entities.end() && TYPE_FROM_HANDLE( *start_rit ) == MBENTITYSET )
            break;
 
        start_rit = end_rit;
    }
 
    // Pack MBMAXTYPE to indicate end of ranges
    buff->check_space( sizeof( int ) );
    PACK_INT( buff->buff_ptr, ( (int)MBMAXTYPE ) );
 
    buff->set_stored_size();
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::build_sharedhps_list( const EntityHandle entity,
                                              const unsigned char pstatus,
                                              const int
#ifndef NDEBUG
                                                  sharedp
#endif
                                              ,
                                              const std::set< unsigned int >& procs,
                                              unsigned int& num_ents,
                                              int* tmp_procs,
                                              EntityHandle* tmp_handles )
{
    num_ents = 0;
    unsigned char pstat;
    ErrorCode result = get_sharing_data( entity, tmp_procs, tmp_handles, pstat, num_ents );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
    assert( pstat == pstatus );
 
    // Build shared proc/handle lists
    // Start with multi-shared, since if it is the owner will be first
    if( pstatus & PSTATUS_MULTISHARED )
    {
    }
    else if( pstatus & PSTATUS_NOT_OWNED )
    {
        // If not multishared and not owned, other sharing proc is owner, put that
        // one first
        assert( "If not owned, I should be shared too" && pstatus & PSTATUS_SHARED && 1 == num_ents );
        tmp_procs[1]   = procConfig.proc_rank();
        tmp_handles[1] = entity;
        num_ents       = 2;
    }
    else if( pstatus & PSTATUS_SHARED )
    {
        // If not multishared and owned, I'm owner
        assert( "shared and owned, should be only 1 sharing proc" && 1 == num_ents );
        tmp_procs[1]   = tmp_procs[0];
        tmp_procs[0]   = procConfig.proc_rank();
        tmp_handles[1] = tmp_handles[0];
        tmp_handles[0] = entity;
        num_ents       = 2;
    }
    else
    {
        // Not shared yet, just add owner (me)
        tmp_procs[0]   = procConfig.proc_rank();
        tmp_handles[0] = entity;
        num_ents       = 1;
    }
 
#ifndef NDEBUG
    int tmp_ps = num_ents;
#endif
 
    // Now add others, with zero handle for now
    for( std::set< unsigned int >::iterator sit = procs.begin(); sit != procs.end(); ++sit )
    {
#ifndef NDEBUG
        if( tmp_ps && std::find( tmp_procs, tmp_procs + tmp_ps, *sit ) != tmp_procs + tmp_ps )
        {
            std::cerr << "Trouble with something already in shared list on proc " << procConfig.proc_rank()
                      << ". Entity:" << std::endl;
            list_entities( &entity, 1 );
            std::cerr << "pstatus = " << (int)pstatus << ", sharedp = " << sharedp << std::endl;
            std::cerr << "tmp_ps = ";
            for( int i = 0; i < tmp_ps; i++ )
                std::cerr << tmp_procs[i] << " ";
            std::cerr << std::endl;
            std::cerr << "procs = ";
            for( std::set< unsigned int >::iterator sit2 = procs.begin(); sit2 != procs.end(); ++sit2 )
                std::cerr << *sit2 << " ";
            assert( false );
        }
#endif
        tmp_procs[num_ents]   = *sit;
        tmp_handles[num_ents] = 0;
        num_ents++;
    }
 
    // Put -1 after procs and 0 after handles
    if( MAX_SHARING_PROCS > num_ents )
    {
        tmp_procs[num_ents]   = -1;
        tmp_handles[num_ents] = 0;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::pack_entity_seq( const int nodes_per_entity,
                                         const bool store_remote_handles,
                                         const int to_proc,
                                         Range& these_ents,
                                         std::vector< EntityHandle >& entities_vec,
                                         Buffer* buff )
{
    int tmp_space = 3 * sizeof( int ) + nodes_per_entity * these_ents.size() * sizeof( EntityHandle );
    buff->check_space( tmp_space );
 
    // Pack the entity type
    PACK_INT( buff->buff_ptr, ( (int)TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
 
    // Pack # ents
    PACK_INT( buff->buff_ptr, these_ents.size() );
 
    // Pack the nodes per entity
    PACK_INT( buff->buff_ptr, nodes_per_entity );
    myDebug->tprintf( 3, "after some pack int  %d \n", buff->get_current_size() );
 
    // Pack the connectivity
    std::vector< EntityHandle > connect;
    ErrorCode result = MB_SUCCESS;
    for( Range::const_iterator rit = these_ents.begin(); rit != these_ents.end(); ++rit )
    {
        connect.clear();
        result = mbImpl->get_connectivity( &( *rit ), 1, connect, false );MB_CHK_SET_ERR( result, "Failed to get connectivity" );
        assert( (int)connect.size() == nodes_per_entity );
        result =
            get_remote_handles( store_remote_handles, &connect[0], &connect[0], connect.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
        PACK_EH( buff->buff_ptr, &connect[0], connect.size() );
    }
 
    myDebug->tprintf( 3, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
                      CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
 
    return result;
}
 
ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
                                            EntityHandle* from_vec,
                                            EntityHandle* to_vec_tmp,
                                            int num_ents,
                                            int to_proc,
                                            const std::vector< EntityHandle >& new_ents )
{
    // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE RANGE-BASED VERSION, NO REUSE
    // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE
    // OTHER VERSION TOO!!!
    if( 0 == num_ents ) return MB_SUCCESS;
 
    // Use a local destination ptr in case we're doing an in-place copy
    std::vector< EntityHandle > tmp_vector;
    EntityHandle* to_vec = to_vec_tmp;
    if( to_vec == from_vec )
    {
        tmp_vector.resize( num_ents );
        to_vec = &tmp_vector[0];
    }
 
    if( !store_remote_handles )
    {
        int err;
        // In this case, substitute position in new_ents list
        for( int i = 0; i < num_ents; i++ )
        {
            int ind = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
            assert( new_ents[ind] == from_vec[i] );
            to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
            assert( to_vec[i] != 0 && !err && -1 != ind );
        }
    }
    else
    {
        Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
        ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
 
        // Get single-proc destination handles and shared procs
        std::vector< int > sharing_procs( num_ents );
        result = mbImpl->tag_get_data( shh_tag, from_vec, num_ents, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
        result = mbImpl->tag_get_data( shp_tag, from_vec, num_ents, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
        for( int j = 0; j < num_ents; j++ )
        {
            if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
        }
 
        EntityHandle tmp_handles[MAX_SHARING_PROCS];
        int tmp_procs[MAX_SHARING_PROCS];
        int i;
        // Go through results, and for 0-valued ones, look for multiple shared proc
        for( i = 0; i < num_ents; i++ )
        {
            if( !to_vec[i] )
            {
                result = mbImpl->tag_get_data( shps_tag, from_vec + i, 1, tmp_procs );
                if( MB_SUCCESS == result )
                {
                    for( int j = 0; j < MAX_SHARING_PROCS; j++ )
                    {
                        if( -1 == tmp_procs[j] )
                            break;
                        else if( tmp_procs[j] == to_proc )
                        {
                            result = mbImpl->tag_get_data( shhs_tag, from_vec + i, 1, tmp_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
                            to_vec[i] = tmp_handles[j];
                            assert( to_vec[i] );
                            break;
                        }
                    }
                }
                if( !to_vec[i] )
                {
                    int j = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
                    if( (int)new_ents.size() == j )
                    {
                        std::cout << "Failed to find new entity in send list, proc " << procConfig.proc_rank()
                                  << std::endl;
                        for( int k = 0; k <= num_ents; k++ )
                            std::cout << k << ": " << from_vec[k] << " " << to_vec[k] << std::endl;
                        MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
                    }
                    int err;
                    to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
                    if( err )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
                    }
                }
            }
        }
    }
 
    // memcpy over results if from_vec and to_vec are the same
    if( to_vec_tmp == from_vec ) memcpy( from_vec, to_vec, num_ents * sizeof( EntityHandle ) );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
                                            const Range& from_range,
                                            EntityHandle* to_vec,
                                            int to_proc,
                                            const std::vector< EntityHandle >& new_ents )
{
    // NOTE: THIS IMPLEMENTATION IS JUST LIKE THE VECTOR-BASED VERSION, NO REUSE
    // AT THIS TIME, SO IF YOU FIX A BUG IN THIS VERSION, IT MAY BE IN THE
    // OTHER VERSION TOO!!!
    if( from_range.empty() ) return MB_SUCCESS;
 
    if( !store_remote_handles )
    {
        int err;
        // In this case, substitute position in new_ents list
        Range::iterator rit;
        unsigned int i;
        for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
        {
            int ind = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
            assert( new_ents[ind] == *rit );
            to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
            assert( to_vec[i] != 0 && !err && -1 != ind );
        }
    }
    else
    {
        Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
        ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
 
        // Get single-proc destination handles and shared procs
        std::vector< int > sharing_procs( from_range.size() );
        result = mbImpl->tag_get_data( shh_tag, from_range, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
        result = mbImpl->tag_get_data( shp_tag, from_range, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
        for( unsigned int j = 0; j < from_range.size(); j++ )
        {
            if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
        }
 
        EntityHandle tmp_handles[MAX_SHARING_PROCS];
        int tmp_procs[MAX_SHARING_PROCS];
        // Go through results, and for 0-valued ones, look for multiple shared proc
        Range::iterator rit;
        unsigned int i;
        for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
        {
            if( !to_vec[i] )
            {
                result = mbImpl->tag_get_data( shhs_tag, &( *rit ), 1, tmp_handles );
                if( MB_SUCCESS == result )
                {
                    result = mbImpl->tag_get_data( shps_tag, &( *rit ), 1, tmp_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
                    for( int j = 0; j < MAX_SHARING_PROCS; j++ )
                        if( tmp_procs[j] == to_proc )
                        {
                            to_vec[i] = tmp_handles[j];
                            break;
                        }
                }
 
                if( !to_vec[i] )
                {
                    int j = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
                    if( (int)new_ents.size() == j )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
                    }
                    int err;
                    to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
                    if( err )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
                    }
                }
            }
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
                                            const Range& from_range,
                                            Range& to_range,
                                            int to_proc,
                                            const std::vector< EntityHandle >& new_ents )
{
    std::vector< EntityHandle > to_vector( from_range.size() );
 
    ErrorCode result = get_remote_handles( store_remote_handles, from_range, &to_vector[0], to_proc, new_ents );MB_CHK_SET_ERR( result, "Failed to get remote handles" );
    std::copy( to_vector.begin(), to_vector.end(), range_inserter( to_range ) );
    return result;
}
 
ErrorCode ParallelComm::unpack_entities( unsigned char*& buff_ptr,
                                         const bool store_remote_handles,
                                         const int /*from_ind*/,
                                         const bool is_iface,
                                         std::vector< std::vector< EntityHandle > >& L1hloc,
                                         std::vector< std::vector< EntityHandle > >& L1hrem,
                                         std::vector< std::vector< int > >& L1p,
                                         std::vector< EntityHandle >& L2hloc,
                                         std::vector< EntityHandle >& L2hrem,
                                         std::vector< unsigned int >& L2p,
                                         std::vector< EntityHandle >& new_ents,
                                         const bool created_iface )
{
    // General algorithm:
    // - unpack # entities
    // - save start of remote handle info, then scan forward to entity definition data
    // - for all vertices or entities w/ same # verts:
    //   . get entity type, num ents, and (if !vert) # verts
    //   . for each ent:
    //      o get # procs/handles in remote handle info
    //      o if # procs/handles > 2, check for already-created entity:
    //        x get index of owner proc (1st in proc list), resize L1 list if nec
    //        x look for already-arrived entity in L2 by owner handle
    //      o if no existing entity:
    //        x if iface, look for existing entity with same connect & type
    //        x if none found, create vertex or element
    //        x if !iface & multi-shared, save on L2
    //        x if !iface, put new entity on new_ents list
    //      o update proc/handle, pstatus tags, adjusting to put owner first if iface
    //      o if !iface, save new handle on L1 for all sharing procs
 
    // Lists of handles/procs to return to sending/other procs
    // L1hloc[p], L1hrem[p]: handle pairs [h, h'], where h is the local proc handle
    //         and h' is either the remote proc handle (if that is known) or
    //         the owner proc handle (otherwise);
    // L1p[p]: indicates whether h is remote handle (= -1) or owner (rank of owner)
    // L2hloc, L2hrem: local/remote handles for entities shared by > 2 procs;
    //         remote handles are on owning proc
    // L2p: owning procs for handles in L2hrem
 
    ErrorCode result;
    bool done         = false;
    ReadUtilIface* ru = NULL;
 
    result = mbImpl->query_interface( ru );MB_CHK_SET_ERR( result, "Failed to get ReadUtilIface" );
 
    // 1. # entities = E
    int num_ents             = 0;
    unsigned char* buff_save = buff_ptr;
    int i, j;
 
    if( store_remote_handles )
    {
        UNPACK_INT( buff_ptr, num_ents );
 
        buff_save = buff_ptr;
 
        // Save place where remote handle info starts, then scan forward to ents
        for( i = 0; i < num_ents; i++ )
        {
            UNPACK_INT( buff_ptr, j );
            if( j < 0 )
            {
                std::cout << "Should be non-negative # proc/handles.";
                return MB_FAILURE;
            }
 
            buff_ptr += j * ( sizeof( int ) + sizeof( EntityHandle ) );
        }
    }
 
    std::vector< EntityHandle > msg_ents;
 
    while( !done )
    {
        EntityType this_type = MBMAXTYPE;
        UNPACK_TYPE( buff_ptr, this_type );
        assert( this_type != MBENTITYSET );
 
        // MBMAXTYPE signifies end of entities data
        if( MBMAXTYPE == this_type ) break;
 
        // Get the number of ents
        int num_ents2, verts_per_entity = 0;
        UNPACK_INT( buff_ptr, num_ents2 );
 
        // Unpack the nodes per entity
        if( MBVERTEX != this_type && num_ents2 )
        {
            UNPACK_INT( buff_ptr, verts_per_entity );
        }
 
        std::vector< int > ps( MAX_SHARING_PROCS, -1 );
        std::vector< EntityHandle > hs( MAX_SHARING_PROCS, 0 );
        for( int e = 0; e < num_ents2; e++ )
        {
            // Check for existing entity, otherwise make new one
            EntityHandle new_h = 0;
            EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
            double coords[3];
            int num_ps = -1;
 
            //=======================================
            // Unpack all the data at once, to make sure the buffer pointers
            // are tracked correctly
            //=======================================
            if( store_remote_handles )
            {
                // Pointers to other procs/handles
                UNPACK_INT( buff_save, num_ps );
                if( 0 >= num_ps )
                {
                    std::cout << "Shouldn't ever be fewer than 1 procs here." << std::endl;
                    return MB_FAILURE;
                }
 
                UNPACK_INTS( buff_save, &ps[0], num_ps );
                UNPACK_EH( buff_save, &hs[0], num_ps );
            }
 
            if( MBVERTEX == this_type )
            {
                UNPACK_DBLS( buff_ptr, coords, 3 );
            }
            else
            {
                assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
                UNPACK_EH( buff_ptr, connect, verts_per_entity );
 
                // Update connectivity to local handles
                result = get_local_handles( connect, verts_per_entity, msg_ents );MB_CHK_SET_ERR( result, "Failed to get local handles" );
            }
 
            //=======================================
            // Now, process that data; begin by finding an identical
            // entity, if there is one
            //=======================================
            if( store_remote_handles )
            {
                result = find_existing_entity( is_iface, ps[0], hs[0], num_ps, connect, verts_per_entity, this_type,
                                               L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
            }
 
            //=======================================
            // If we didn't find one, we'll have to create one
            //=======================================
            bool created_here = false;
            if( !new_h && !is_iface )
            {
                if( MBVERTEX == this_type )
                {
                    // Create a vertex
                    result = mbImpl->create_vertex( coords, new_h );MB_CHK_SET_ERR( result, "Failed to make new vertex" );
                }
                else
                {
                    // Create the element
                    result = mbImpl->create_element( this_type, connect, verts_per_entity, new_h );MB_CHK_SET_ERR( result, "Failed to make new element" );
 
                    // Update adjacencies
                    result = ru->update_adjacencies( new_h, 1, verts_per_entity, connect );MB_CHK_SET_ERR( result, "Failed to update adjacencies" );
                }
 
                // Should have a new handle now
                assert( new_h );
 
                created_here = true;
            }
 
            //=======================================
            // Take care of sharing data
            //=======================================
 
            // Need to save entities found in order, for interpretation of
            // later parts of this message
            if( !is_iface )
            {
                assert( new_h );
                msg_ents.push_back( new_h );
            }
 
            if( created_here ) new_ents.push_back( new_h );
 
            if( new_h && store_remote_handles )
            {
                unsigned char new_pstat = 0x0;
                if( is_iface )
                {
                    new_pstat = PSTATUS_INTERFACE;
                    // Here, lowest rank proc should be first
                    int idx = std::min_element( &ps[0], &ps[0] + num_ps ) - &ps[0];
                    if( idx )
                    {
                        std::swap( ps[0], ps[idx] );
                        std::swap( hs[0], hs[idx] );
                    }
                    // Set ownership based on lowest rank; can't be in update_remote_data, because
                    // there we don't know whether it resulted from ghosting or not
                    if( ( num_ps > 1 && ps[0] != (int)rank() ) ) new_pstat |= PSTATUS_NOT_OWNED;
                }
                else if( created_here )
                {
                    if( created_iface )
                        new_pstat = PSTATUS_NOT_OWNED;
                    else
                        new_pstat = PSTATUS_GHOST | PSTATUS_NOT_OWNED;
                }
 
                // Update sharing data and pstatus, adjusting order if iface
                result = update_remote_data( new_h, &ps[0], &hs[0], num_ps, new_pstat );MB_CHK_SET_ERR( result, "unpack_entities" );
 
                // If a new multi-shared entity, save owner for subsequent lookup in L2 lists
                if( store_remote_handles && !is_iface && num_ps > 2 )
                {
                    L2hrem.push_back( hs[0] );
                    L2hloc.push_back( new_h );
                    L2p.push_back( ps[0] );
                }
 
                // Need to send this new handle to all sharing procs
                if( !is_iface )
                {
                    for( j = 0; j < num_ps; j++ )
                    {
                        if( ps[j] == (int)procConfig.proc_rank() ) continue;
                        int idx = get_buffers( ps[j] );
                        if( idx == (int)L1hloc.size() )
                        {
                            L1hloc.resize( idx + 1 );
                            L1hrem.resize( idx + 1 );
                            L1p.resize( idx + 1 );
                        }
 
                        // Don't bother adding if it's already in the list
                        std::vector< EntityHandle >::iterator vit =
                            std::find( L1hloc[idx].begin(), L1hloc[idx].end(), new_h );
                        if( vit != L1hloc[idx].end() )
                        {
                            // If it's in the list but remote handle isn't known but we know
                            // it, replace in the list
                            if( L1p[idx][vit - L1hloc[idx].begin()] != -1 && hs[j] )
                            {
                                L1hrem[idx][vit - L1hloc[idx].begin()] = hs[j];
                                L1p[idx][vit - L1hloc[idx].begin()]    = -1;
                            }
                            else
                                continue;
                        }
                        else
                        {
                            if( !hs[j] )
                            {
                                assert( -1 != ps[0] && num_ps > 2 );
                                L1p[idx].push_back( ps[0] );
                                L1hrem[idx].push_back( hs[0] );
                            }
                            else
                            {
                                assert(
                                    "either this remote handle isn't in the remote list, or "
                                    "it's for another proc" &&
                                    ( std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) == L1hrem[idx].end() ||
                                      L1p[idx][std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) -
                                               L1hrem[idx].begin()] != -1 ) );
                                L1p[idx].push_back( -1 );
                                L1hrem[idx].push_back( hs[j] );
                            }
                            L1hloc[idx].push_back( new_h );
                        }
                    }
                }
 
                assert( "Shouldn't be here for non-shared entities" && -1 != num_ps );
                std::fill( &ps[0], &ps[num_ps], -1 );
                std::fill( &hs[0], &hs[num_ps], 0 );
            }
        }
 
        myDebug->tprintf( 4, "Unpacked %d ents of type %s", num_ents2, CN::EntityTypeName( this_type ) );
    }
 
    myDebug->tprintf( 4, "Done unpacking entities.\n" );
 
    // Need to sort here, to enable searching
    std::sort( new_ents.begin(), new_ents.end() );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::print_buffer( unsigned char* buff_ptr, int mesg_tag, int from_proc, bool sent )
{
    std::cerr << procConfig.proc_rank();
    if( sent )
        std::cerr << " sent";
    else
        std::cerr << " received";
    std::cerr << " message type " << mesg_tag << " to/from proc " << from_proc << "; contents:" << std::endl;
 
    int msg_length, num_ents;
    unsigned char* orig_ptr = buff_ptr;
    UNPACK_INT( buff_ptr, msg_length );
    std::cerr << msg_length << " bytes..." << std::endl;
 
    if( MB_MESG_ENTS_SIZE == mesg_tag || MB_MESG_ENTS_LARGE == mesg_tag )
    {
        // 1. # entities = E
        int i, j, k;
        std::vector< int > ps;
        std::vector< EntityHandle > hs;
 
        UNPACK_INT( buff_ptr, num_ents );
        std::cerr << num_ents << " entities..." << std::endl;
 
        // Save place where remote handle info starts, then scan forward to ents
        for( i = 0; i < num_ents; i++ )
        {
            UNPACK_INT( buff_ptr, j );
            if( 0 > j ) return MB_FAILURE;
            ps.resize( j );
            hs.resize( j );
            std::cerr << "Entity " << i << ", # procs = " << j << std::endl;
            UNPACK_INTS( buff_ptr, &ps[0], j );
            UNPACK_EH( buff_ptr, &hs[0], j );
            std::cerr << "   Procs: ";
            for( k = 0; k < j; k++ )
                std::cerr << ps[k] << " ";
            std::cerr << std::endl;
            std::cerr << "   Handles: ";
            for( k = 0; k < j; k++ )
                std::cerr << hs[k] << " ";
            std::cerr << std::endl;
 
            if( buff_ptr - orig_ptr > msg_length )
            {
                std::cerr << "End of buffer..." << std::endl;
                std::cerr.flush();
                return MB_FAILURE;
            }
        }
 
        while( true )
        {
            EntityType this_type = MBMAXTYPE;
            UNPACK_TYPE( buff_ptr, this_type );
            assert( this_type != MBENTITYSET );
 
            // MBMAXTYPE signifies end of entities data
            if( MBMAXTYPE == this_type ) break;
 
            // Get the number of ents
            int num_ents2, verts_per_entity = 0;
            UNPACK_INT( buff_ptr, num_ents2 );
 
            // Unpack the nodes per entity
            if( MBVERTEX != this_type && num_ents2 )
            {
                UNPACK_INT( buff_ptr, verts_per_entity );
            }
 
            std::cerr << "Type: " << CN::EntityTypeName( this_type ) << "; num_ents = " << num_ents2;
            if( MBVERTEX != this_type ) std::cerr << "; verts_per_ent = " << verts_per_entity;
            std::cerr << std::endl;
            if( num_ents2 < 0 || num_ents2 > msg_length )
            {
                std::cerr << "Wrong number of entities, returning." << std::endl;
                return MB_FAILURE;
            }
 
            for( int e = 0; e < num_ents2; e++ )
            {
                // Check for existing entity, otherwise make new one
                if( MBVERTEX == this_type )
                {
                    double coords[3];
                    UNPACK_DBLS( buff_ptr, coords, 3 );
                    std::cerr << "xyz = " << coords[0] << ", " << coords[1] << ", " << coords[2] << std::endl;
                }
                else
                {
                    EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
                    assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
                    UNPACK_EH( buff_ptr, connect, verts_per_entity );
 
                    // Update connectivity to local handles
                    std::cerr << "Connectivity: ";
                    for( k = 0; k < verts_per_entity; k++ )
                        std::cerr << connect[k] << " ";
                    std::cerr << std::endl;
                }
 
                if( buff_ptr - orig_ptr > msg_length )
                {
                    std::cerr << "End of buffer..." << std::endl;
                    std::cerr.flush();
                    return MB_FAILURE;
                }
            }
        }
    }
    else if( MB_MESG_REMOTEH_SIZE == mesg_tag || MB_MESG_REMOTEH_LARGE == mesg_tag )
    {
        UNPACK_INT( buff_ptr, num_ents );
        std::cerr << num_ents << " entities..." << std::endl;
        if( 0 > num_ents || num_ents > msg_length )
        {
            std::cerr << "Wrong number of entities, returning." << std::endl;
            return MB_FAILURE;
        }
        std::vector< EntityHandle > L1hloc( num_ents ), L1hrem( num_ents );
        std::vector< int > L1p( num_ents );
        UNPACK_INTS( buff_ptr, &L1p[0], num_ents );
        UNPACK_EH( buff_ptr, &L1hrem[0], num_ents );
        UNPACK_EH( buff_ptr, &L1hloc[0], num_ents );
        std::cerr << num_ents << " Entity pairs; hremote/hlocal/proc: " << std::endl;
        for( int i = 0; i < num_ents; i++ )
        {
            EntityType etype = TYPE_FROM_HANDLE( L1hloc[i] );
            std::cerr << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hrem[i] ) << ", "
                      << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hloc[i] ) << ", " << L1p[i] << std::endl;
        }
 
        if( buff_ptr - orig_ptr > msg_length )
        {
            std::cerr << "End of buffer..." << std::endl;
            std::cerr.flush();
            return MB_FAILURE;
        }
    }
    else if( mesg_tag == MB_MESG_TAGS_SIZE || mesg_tag == MB_MESG_TAGS_LARGE )
    {
        int num_tags, dum1, data_type, tag_size;
        UNPACK_INT( buff_ptr, num_tags );
        std::cerr << "Number of tags = " << num_tags << std::endl;
        for( int i = 0; i < num_tags; i++ )
        {
            std::cerr << "Tag " << i << ":" << std::endl;
            UNPACK_INT( buff_ptr, tag_size );
            UNPACK_INT( buff_ptr, dum1 );
            UNPACK_INT( buff_ptr, data_type );
            std::cerr << "Tag size, type, data type = " << tag_size << ", " << dum1 << ", " << data_type << std::endl;
            UNPACK_INT( buff_ptr, dum1 );
            std::cerr << "Default value size = " << dum1 << std::endl;
            buff_ptr += dum1;
            UNPACK_INT( buff_ptr, dum1 );
            std::string name( (char*)buff_ptr, dum1 );
            std::cerr << "Tag name = " << name.c_str() << std::endl;
            buff_ptr += dum1;
            UNPACK_INT( buff_ptr, num_ents );
            std::cerr << "Number of ents = " << num_ents << std::endl;
            std::vector< EntityHandle > tmp_buff( num_ents );
            UNPACK_EH( buff_ptr, &tmp_buff[0], num_ents );
            int tot_length = 0;
            for( int j = 0; j < num_ents; j++ )
            {
                EntityType etype = TYPE_FROM_HANDLE( tmp_buff[j] );
                std::cerr << CN::EntityTypeName( etype ) << " " << ID_FROM_HANDLE( tmp_buff[j] ) << ", tag = ";
                if( tag_size == MB_VARIABLE_LENGTH )
                {
                    UNPACK_INT( buff_ptr, dum1 );
                    tot_length += dum1;
                    std::cerr << "(variable, length = " << dum1 << ")" << std::endl;
                }
                else if( data_type == MB_TYPE_DOUBLE )
                {
                    double dum_dbl;
                    UNPACK_DBL( buff_ptr, dum_dbl );
                    std::cerr << dum_dbl << std::endl;
                }
                else if( data_type == MB_TYPE_INTEGER )
                {
                    int dum_int;
                    UNPACK_INT( buff_ptr, dum_int );
                    std::cerr << dum_int << std::endl;
                }
                else if( data_type == MB_TYPE_OPAQUE )
                {
                    std::cerr << "(opaque)" << std::endl;
                    buff_ptr += tag_size;
                }
                else if( data_type == MB_TYPE_HANDLE )
                {
                    EntityHandle dum_eh;
                    UNPACK_EH( buff_ptr, &dum_eh, 1 );
                    std::cerr << dum_eh << std::endl;
                }
                else if( data_type == MB_TYPE_BIT )
                {
                    std::cerr << "(bit)" << std::endl;
                    buff_ptr += tag_size;
                }
            }
            if( tag_size == MB_VARIABLE_LENGTH ) buff_ptr += tot_length;
        }
    }
    else
    {
        assert( false );
        return MB_FAILURE;
    }
 
    std::cerr.flush();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::list_entities( const EntityHandle* ents, int num_ents )
{
    if( NULL == ents )
    {
        Range shared_ents;
        std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) );
        shared_ents.print( "Shared entities:\n" );
        return MB_SUCCESS;
    }
 
    unsigned char pstat;
    EntityHandle tmp_handles[MAX_SHARING_PROCS];
    int tmp_procs[MAX_SHARING_PROCS];
    unsigned int num_ps;
    ErrorCode result;
 
    for( int i = 0; i < num_ents; i++ )
    {
        result = mbImpl->list_entities( ents + i, 1 );MB_CHK_ERR( result );
        double coords[3];
        result = mbImpl->get_coords( ents + i, 1, coords );
        std::cout << " coords: " << coords[0] << " " << coords[1] << " " << coords[2] << "\n";
 
        result = get_sharing_data( ents[i], tmp_procs, tmp_handles, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
 
        std::cout << "Pstatus: ";
        if( !num_ps )
            std::cout << "local " << std::endl;
        else
        {
            if( pstat & PSTATUS_NOT_OWNED ) std::cout << "NOT_OWNED; ";
            if( pstat & PSTATUS_SHARED ) std::cout << "SHARED; ";
            if( pstat & PSTATUS_MULTISHARED ) std::cout << "MULTISHARED; ";
            if( pstat & PSTATUS_INTERFACE ) std::cout << "INTERFACE; ";
            if( pstat & PSTATUS_GHOST ) std::cout << "GHOST; ";
            std::cout << std::endl;
            for( unsigned int j = 0; j < num_ps; j++ )
            {
                std::cout << "  proc " << tmp_procs[j] << " id (handle) " << mbImpl->id_from_handle( tmp_handles[j] )
                          << "(" << tmp_handles[j] << ")" << std::endl;
            }
        }
        std::cout << std::endl;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::list_entities( const Range& ents )
{
    for( Range::iterator rit = ents.begin(); rit != ents.end(); ++rit )
        list_entities( &( *rit ), 1 );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::update_remote_data( Range& local_range,
                                            Range& remote_range,
                                            int other_proc,
                                            const unsigned char add_pstat )
{
    Range::iterator rit, rit2;
    ErrorCode result = MB_SUCCESS;
 
    // For each pair of local/remote handles:
    for( rit = local_range.begin(), rit2 = remote_range.begin(); rit != local_range.end(); ++rit, ++rit2 )
    {
        result = update_remote_data( *rit, &other_proc, &( *rit2 ), 1, add_pstat );MB_CHK_ERR( result );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::update_remote_data( const EntityHandle new_h,
                                            const int* ps,
                                            const EntityHandle* hs,
                                            const int num_ps,
                                            const unsigned char add_pstat
                                            // The following lines left in for future debugging, at least until I trust
                                            // this function; tjt, 10/4/2013
                                            //                                           , int *new_ps,
                                            //                                           EntityHandle *new_hs,
                                            //                                           int &new_numps,
                                            //                                           unsigned char &new_pstat
)
{
    // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0
    // in this function, so no need to initialize; sharing data does not include
    // this proc if shared with only one other
 
    // Following variables declared here to avoid compiler errors
    int new_numps;
    unsigned char new_pstat;
    std::vector< int > new_ps( MAX_SHARING_PROCS, -1 );
    std::vector< EntityHandle > new_hs( MAX_SHARING_PROCS, 0 );
 
    new_numps        = 0;
    ErrorCode result = get_sharing_data( new_h, &new_ps[0], &new_hs[0], new_pstat, new_numps );MB_CHK_SET_ERR( result, "Failed to get sharing data in update_remote_data" );
    int num_exist = new_numps;
 
    // Add new pstat info to the flag
    new_pstat |= add_pstat;
 
    /*
    #define plist(str, lst, siz)                                          \
        std::cout << str << "(";                                          \
        for (int i = 0; i < (int)siz; i++) std::cout << lst[i] << " ";    \
        std::cout << ") ";                                                \
 
        std::cout << "update_remote_data: rank = " << rank() << ", new_h = " << new_h << std::endl;
        std::string ostr;
        plist("ps", ps, num_ps);
        plist("hs", hs, num_ps);
        print_pstatus(add_pstat, ostr);
        std::cout << ", add_pstat = " << ostr.c_str() << std::endl;
        plist("tag_ps", new_ps, new_numps);
        plist("tag_hs", new_hs, new_numps);
        assert(new_numps <= size());
        print_pstatus(new_pstat, ostr);
        std::cout << ", tag_pstat=" << ostr.c_str() << std::endl;
    */
 
#ifndef NDEBUG
    {
        // Check for duplicates in proc list
        std::set< unsigned int > dumprocs;
        unsigned int dp = 0;
        for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
            dumprocs.insert( ps[dp] );
        assert( dp == dumprocs.size() );
    }
#endif
 
    // If only one sharer and I'm the owner, insert myself in the list;
    // otherwise, my data is checked at the end
    if( 1 == new_numps && !( new_pstat & PSTATUS_NOT_OWNED ) )
    {
        new_hs[1] = new_hs[0];
        new_ps[1] = new_ps[0];
        new_hs[0] = new_h;
        new_ps[0] = rank();
        new_numps = 2;
    }
 
    // Now put passed-in data onto lists
    int idx;
    for( int i = 0; i < num_ps; i++ )
    {
        idx = std::find( &new_ps[0], &new_ps[0] + new_numps, ps[i] ) - &new_ps[0];
        if( idx < new_numps )
        {
            if( !new_hs[idx] && hs[i] )
                // h on list is 0 and passed-in h is non-zero, replace it
                new_hs[idx] = hs[i];
            else
                assert( !hs[i] || new_hs[idx] == hs[i] );
        }
        else
        {
            if( new_numps + 1 == MAX_SHARING_PROCS )
            {
                MB_SET_ERR( MB_FAILURE, "Exceeded MAX_SHARING_PROCS for "
                                            << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) << ' '
                                            << ID_FROM_HANDLE( new_h ) << " in process " << rank() );
            }
            new_ps[new_numps] = ps[i];
            new_hs[new_numps] = hs[i];
            new_numps++;
        }
    }
 
    // Add myself, if it isn't there already
    idx = std::find( &new_ps[0], &new_ps[0] + new_numps, rank() ) - &new_ps[0];
    if( idx == new_numps )
    {
        new_ps[new_numps] = rank();
        new_hs[new_numps] = new_h;
        new_numps++;
    }
    else if( !new_hs[idx] && new_numps > 2 )
        new_hs[idx] = new_h;
 
    // Proc list is complete; update for shared, multishared
    if( new_numps > 1 )
    {
        if( new_numps > 2 ) new_pstat |= PSTATUS_MULTISHARED;
        new_pstat |= PSTATUS_SHARED;
    }
 
    /*
        plist("new_ps", new_ps, new_numps);
        plist("new_hs", new_hs, new_numps);
        print_pstatus(new_pstat, ostr);
        std::cout << ", new_pstat=" << ostr.c_str() << std::endl;
        std::cout << std::endl;
    */
 
    result = set_sharing_data( new_h, new_pstat, num_exist, new_numps, &new_ps[0], &new_hs[0] );MB_CHK_SET_ERR( result, "Failed to set sharing data in update_remote_data" );
 
    if( new_pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::update_remote_data_old( const EntityHandle new_h,
                                                const int* ps,
                                                const EntityHandle* hs,
                                                const int num_ps,
                                                const unsigned char add_pstat )
{
    EntityHandle tag_hs[MAX_SHARING_PROCS];
    int tag_ps[MAX_SHARING_PROCS];
    unsigned char pstat;
    // Get initial sharing data; tag_ps and tag_hs get terminated with -1 and 0
    // in this function, so no need to initialize
    unsigned int num_exist;
    ErrorCode result = get_sharing_data( new_h, tag_ps, tag_hs, pstat, num_exist );MB_CHK_ERR( result );
 
#ifndef NDEBUG
    {
        // Check for duplicates in proc list
        std::set< unsigned int > dumprocs;
        unsigned int dp = 0;
        for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
            dumprocs.insert( ps[dp] );
        assert( dp == dumprocs.size() );
    }
#endif
 
    // Add any new sharing data
    bool changed = false;
    int idx;
    if( !num_exist )
    {
        // Just take what caller passed
        memcpy( tag_ps, ps, num_ps * sizeof( int ) );
        memcpy( tag_hs, hs, num_ps * sizeof( EntityHandle ) );
        num_exist = num_ps;
        // If it's only one, hopefully I'm not there yet...
        assert( "I shouldn't be the only proc there." && ( 1 != num_exist || ps[0] != (int)procConfig.proc_rank() ) );
        changed = true;
    }
    else
    {
        for( int i = 0; i < num_ps; i++ )
        {
            idx = std::find( tag_ps, tag_ps + num_exist, ps[i] ) - tag_ps;
            if( idx == (int)num_exist )
            {
                if( num_exist == MAX_SHARING_PROCS )
                {
                    std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) )
                              << ' ' << ID_FROM_HANDLE( new_h ) << " in process " << proc_config().proc_rank()
                              << std::endl;
                    std::cerr.flush();
                    MPI_Abort( proc_config().proc_comm(), 66 );
                }
 
                // If there's only 1 sharing proc, and it's not me, then
                // we'll end up with 3; add me to the front
                if( !i && num_ps == 1 && num_exist == 1 && ps[0] != (int)procConfig.proc_rank() )
                {
                    int j = 1;
                    // If I own this entity, put me at front, otherwise after first
                    if( !( pstat & PSTATUS_NOT_OWNED ) )
                    {
                        tag_ps[1] = tag_ps[0];
                        tag_hs[1] = tag_hs[0];
                        j         = 0;
                    }
                    tag_ps[j] = procConfig.proc_rank();
                    tag_hs[j] = new_h;
                    num_exist++;
                }
 
                tag_ps[num_exist] = ps[i];
                tag_hs[num_exist] = hs[i];
                num_exist++;
                changed = true;
            }
            else if( 0 == tag_hs[idx] )
            {
                tag_hs[idx] = hs[i];
                changed     = true;
            }
            else if( 0 != hs[i] )
            {
                assert( hs[i] == tag_hs[idx] );
            }
        }
    }
 
    // Adjust for interface layer if necessary
    if( add_pstat & PSTATUS_INTERFACE )
    {
        idx = std::min_element( tag_ps, tag_ps + num_exist ) - tag_ps;
        if( idx )
        {
            int tag_proc       = tag_ps[idx];
            tag_ps[idx]        = tag_ps[0];
            tag_ps[0]          = tag_proc;
            EntityHandle tag_h = tag_hs[idx];
            tag_hs[idx]        = tag_hs[0];
            tag_hs[0]          = tag_h;
            changed            = true;
            if( tag_ps[0] != (int)procConfig.proc_rank() ) pstat |= PSTATUS_NOT_OWNED;
        }
    }
 
    if( !changed ) return MB_SUCCESS;
 
    assert( "interface entities should have > 1 proc" && ( !( add_pstat & PSTATUS_INTERFACE ) || num_exist > 1 ) );
    assert( "ghost entities should have > 1 proc" && ( !( add_pstat & PSTATUS_GHOST ) || num_exist > 1 ) );
 
    // If it's multi-shared and we created the entity in this unpack,
    // local handle probably isn't in handle list yet
    if( num_exist > 2 )
    {
        idx = std::find( tag_ps, tag_ps + num_exist, procConfig.proc_rank() ) - tag_ps;
        assert( idx < (int)num_exist );
        if( !tag_hs[idx] ) tag_hs[idx] = new_h;
    }
 
    int tag_p;
    EntityHandle tag_h;
 
    // Update pstat
    pstat |= add_pstat;
 
    if( num_exist > 2 )
        pstat |= ( PSTATUS_MULTISHARED | PSTATUS_SHARED );
    else if( num_exist > 0 )
        pstat |= PSTATUS_SHARED;
 
    //    compare_remote_data(new_h, num_ps, hs, ps, add_pstat,
    //                        num_exist, tag_hs, tag_ps, pstat);
 
    // Reset single shared proc/handle if was shared and moving to multi-shared
    if( num_exist > 2 && !( pstat & PSTATUS_MULTISHARED ) && ( pstat & PSTATUS_SHARED ) )
    {
        // Must remove sharedp/h first, which really means set to default value
        tag_p  = -1;
        result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, &tag_p );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
        tag_h  = 0;
        result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, &tag_h );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
    }
 
    // Set sharing tags
    if( num_exist > 2 )
    {
        std::fill( tag_ps + num_exist, tag_ps + MAX_SHARING_PROCS, -1 );
        std::fill( tag_hs + num_exist, tag_hs + MAX_SHARING_PROCS, 0 );
        result = mbImpl->tag_set_data( sharedps_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" );
        result = mbImpl->tag_set_data( sharedhs_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" );
 
#ifndef NDEBUG
        {
            // Check for duplicates in proc list
            std::set< unsigned int > dumprocs;
            unsigned int dp = 0;
            for( ; dp < num_exist && -1 != tag_ps[dp]; dp++ )
                dumprocs.insert( tag_ps[dp] );
            assert( dp == dumprocs.size() );
        }
#endif
    }
    else if( num_exist == 2 || num_exist == 1 )
    {
        if( tag_ps[0] == (int)procConfig.proc_rank() )
        {
            assert( 2 == num_exist && tag_ps[1] != (int)procConfig.proc_rank() );
            tag_ps[0] = tag_ps[1];
            tag_hs[0] = tag_hs[1];
        }
        assert( tag_ps[0] != -1 && tag_hs[0] != 0 );
        result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
        result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
    }
 
    // Now set new pstatus
    result = mbImpl->tag_set_data( pstatus_tag(), &new_h, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
 
    if( pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_sharing_data( const Range& entities, std::set< int >& procs, int operation )
{
    // Get the union or intersection of sharing data for multiple entities
    ErrorCode result;
    int sp2[MAX_SHARING_PROCS];
    int num_ps;
    unsigned char pstat;
    std::set< int > tmp_procs;
    procs.clear();
 
    for( Range::const_iterator rit = entities.begin(); rit != entities.end(); ++rit )
    {
        // Get sharing procs
        result = get_sharing_data( *rit, sp2, NULL, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_sharing_data" );
        if( !( pstat & PSTATUS_SHARED ) && Interface::INTERSECT == operation )
        {
            procs.clear();
            return MB_SUCCESS;
        }
 
        if( rit == entities.begin() )
        {
            std::copy( sp2, sp2 + num_ps, std::inserter( procs, procs.begin() ) );
        }
        else
        {
            std::sort( sp2, sp2 + num_ps );
            tmp_procs.clear();
            if( Interface::UNION == operation )
                std::set_union( procs.begin(), procs.end(), sp2, sp2 + num_ps,
                                std::inserter( tmp_procs, tmp_procs.end() ) );
            else if( Interface::INTERSECT == operation )
                std::set_intersection( procs.begin(), procs.end(), sp2, sp2 + num_ps,
                                       std::inserter( tmp_procs, tmp_procs.end() ) );
            else
            {
                assert( "Unknown operation." && false );
                return MB_FAILURE;
            }
            procs.swap( tmp_procs );
        }
        if( Interface::INTERSECT == operation && procs.empty() ) return MB_SUCCESS;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity,
                                          int* ps,
                                          EntityHandle* hs,
                                          unsigned char& pstat,
                                          unsigned int& num_ps )
{
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
    if( pstat & PSTATUS_MULTISHARED )
    {
        result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
        if( hs )
        {
            result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
        }
        num_ps = std::find( ps, ps + MAX_SHARING_PROCS, -1 ) - ps;
    }
    else if( pstat & PSTATUS_SHARED )
    {
        result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
        if( hs )
        {
            result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
            hs[1] = 0;
        }
        // Initialize past end of data
        ps[1]  = -1;
        num_ps = 1;
    }
    else
    {
        ps[0] = -1;
        if( hs ) hs[0] = 0;
        num_ps = 0;
    }
 
    assert( MAX_SHARING_PROCS >= num_ps );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::find_existing_entity( const bool is_iface,
                                              const int owner_p,
                                              const EntityHandle owner_h,
                                              const int num_ps,
                                              const EntityHandle* connect,
                                              const int num_connect,
                                              const EntityType this_type,
                                              std::vector< EntityHandle >& L2hloc,
                                              std::vector< EntityHandle >& L2hrem,
                                              std::vector< unsigned int >& L2p,
                                              EntityHandle& new_h )
{
    new_h = 0;
    if( !is_iface && num_ps > 2 )
    {
        for( unsigned int i = 0; i < L2hrem.size(); i++ )
        {
            if( L2hrem[i] == owner_h && owner_p == (int)L2p[i] )
            {
                new_h = L2hloc[i];
                return MB_SUCCESS;
            }
        }
    }
 
    // If we got here and it's a vertex, we don't need to look further
    if( MBVERTEX == this_type || !connect || !num_connect ) return MB_SUCCESS;
 
    Range tmp_range;
    ErrorCode result = mbImpl->get_adjacencies( connect, num_connect, CN::Dimension( this_type ), false, tmp_range );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
    if( !tmp_range.empty() )
    {
        // Found a corresponding entity - return target
        new_h = *tmp_range.begin();
    }
    else
    {
        new_h = 0;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_local_handles( const Range& remote_handles,
                                           Range& local_handles,
                                           const std::vector< EntityHandle >& new_ents )
{
    std::vector< EntityHandle > rh_vec;
    rh_vec.reserve( remote_handles.size() );
    std::copy( remote_handles.begin(), remote_handles.end(), std::back_inserter( rh_vec ) );
    ErrorCode result = get_local_handles( &rh_vec[0], remote_handles.size(), new_ents );
    std::copy( rh_vec.begin(), rh_vec.end(), range_inserter( local_handles ) );
    return result;
}
 
ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents )
{
    std::vector< EntityHandle > tmp_ents;
    std::copy( new_ents.begin(), new_ents.end(), std::back_inserter( tmp_ents ) );
    return get_local_handles( from_vec, num_ents, tmp_ents );
}
 
ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec,
                                           int num_ents,
                                           const std::vector< EntityHandle >& new_ents )
{
    for( int i = 0; i < num_ents; i++ )
    {
        if( TYPE_FROM_HANDLE( from_vec[i] ) == MBMAXTYPE )
        {
            assert( ID_FROM_HANDLE( from_vec[i] ) < (int)new_ents.size() );
            from_vec[i] = new_ents[ID_FROM_HANDLE( from_vec[i] )];
        }
    }
 
    return MB_SUCCESS;
}
 
/*
template <typename T> void
insert_in_array(T* array, size_t array_size, size_t location, T value)
{
  assert(location + 1 < array_size);
  for (size_t i = array_size - 1; i > location; i--)
    array[i] = array[i - 1];
  array[location] = value;
}
*/
 
ErrorCode ParallelComm::pack_range_map( Range& key_range, EntityHandle val_start, HandleMap& handle_map )
{
    for( Range::const_pair_iterator key_it = key_range.const_pair_begin(); key_it != key_range.const_pair_end();
         ++key_it )
    {
        int tmp_num = ( *key_it ).second - ( *key_it ).first + 1;
        handle_map.insert( ( *key_it ).first, val_start, tmp_num );
        val_start += tmp_num;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::pack_sets( Range& entities, Buffer* buff, const bool store_remote_handles, const int to_proc )
{
    // SETS:
    // . #sets
    // . for each set:
    //   - options[#sets] (unsigned int)
    //   - if (unordered) set range
    //   - else if ordered
    //     . #ents in set
    //     . handles[#ents]
    //   - #parents
    //   - if (#parents) handles[#parents]
    //   - #children
    //   - if (#children) handles[#children]
 
    // Now the sets; assume any sets the application wants to pass are in the entities list
    ErrorCode result;
    Range all_sets = entities.subset_by_type( MBENTITYSET );
 
    int buff_size = estimate_sets_buffer_size( all_sets, store_remote_handles );
    if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate sets buffer size" );
    buff->check_space( buff_size );
 
    // Number of sets
    PACK_INT( buff->buff_ptr, all_sets.size() );
 
    // Options for all sets
    std::vector< unsigned int > options( all_sets.size() );
    Range::iterator rit;
    std::vector< EntityHandle > members;
    int i;
    for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
    {
        result = mbImpl->get_meshset_options( *rit, options[i] );MB_CHK_SET_ERR( result, "Failed to get meshset options" );
    }
    buff->check_space( all_sets.size() * sizeof( unsigned int ) );
    PACK_VOID( buff->buff_ptr, &options[0], all_sets.size() * sizeof( unsigned int ) );
 
    // Pack parallel geometry unique id
    if( !all_sets.empty() )
    {
        Tag uid_tag;
        int n_sets  = all_sets.size();
        bool b_pack = false;
        std::vector< int > id_data( n_sets );
        result =
            mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
 
        result = mbImpl->tag_get_data( uid_tag, all_sets, &id_data[0] );
        if( MB_TAG_NOT_FOUND != result )
        {
            if( MB_SUCCESS != result ) MB_SET_ERR( result, "Failed to get parallel geometry unique ids" );
            for( i = 0; i < n_sets; i++ )
            {
                if( id_data[i] != 0 )
                {
                    b_pack = true;
                    break;
                }
            }
        }
 
        if( b_pack )
        {  // If you find
            buff->check_space( ( n_sets + 1 ) * sizeof( int ) );
            PACK_INT( buff->buff_ptr, n_sets );
            PACK_INTS( buff->buff_ptr, &id_data[0], n_sets );
        }
        else
        {
            buff->check_space( sizeof( int ) );
            PACK_INT( buff->buff_ptr, 0 );
        }
    }
 
    // Vectors/ranges
    std::vector< EntityHandle > entities_vec( entities.size() );
    std::copy( entities.begin(), entities.end(), entities_vec.begin() );
    for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
    {
        members.clear();
        result = mbImpl->get_entities_by_handle( *rit, members );MB_CHK_SET_ERR( result, "Failed to get entities in ordered set" );
        result =
            get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
        buff->check_space( members.size() * sizeof( EntityHandle ) + sizeof( int ) );
        PACK_INT( buff->buff_ptr, members.size() );
        PACK_EH( buff->buff_ptr, &members[0], members.size() );
    }
 
    // Pack parent/child sets
    if( !store_remote_handles )
    {  // Only works not store remote handles
        // Pack numbers of parents/children
        unsigned int tot_pch = 0;
        int num_pch;
        buff->check_space( 2 * all_sets.size() * sizeof( int ) );
        for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
        {
            // Pack parents
            result = mbImpl->num_parent_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num parents" );
            PACK_INT( buff->buff_ptr, num_pch );
            tot_pch += num_pch;
            result = mbImpl->num_child_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num children" );
            PACK_INT( buff->buff_ptr, num_pch );
            tot_pch += num_pch;
        }
 
        // Now pack actual parents/children
        members.clear();
        members.reserve( tot_pch );
        std::vector< EntityHandle > tmp_pch;
        for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
        {
            result = mbImpl->get_parent_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get parents" );
            std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
            tmp_pch.clear();
            result = mbImpl->get_child_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get children" );
            std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
            tmp_pch.clear();
        }
        assert( members.size() == tot_pch );
        if( !members.empty() )
        {
            result = get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc,
                                         entities_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles for set parent/child sets" );
#ifndef NDEBUG
            // Check that all handles are either sets or maxtype
            for( unsigned int __j = 0; __j < members.size(); __j++ )
                assert( ( TYPE_FROM_HANDLE( members[__j] ) == MBMAXTYPE &&
                          ID_FROM_HANDLE( members[__j] ) < (int)entities.size() ) ||
                        TYPE_FROM_HANDLE( members[__j] ) == MBENTITYSET );
#endif
            buff->check_space( members.size() * sizeof( EntityHandle ) );
            PACK_EH( buff->buff_ptr, &members[0], members.size() );
        }
    }
    else
    {
        buff->check_space( 2 * all_sets.size() * sizeof( int ) );
        for( rit = all_sets.begin(); rit != all_sets.end(); ++rit )
        {
            PACK_INT( buff->buff_ptr, 0 );
            PACK_INT( buff->buff_ptr, 0 );
        }
    }
 
    // Pack the handles
    if( store_remote_handles && !all_sets.empty() )
    {
        buff_size = RANGE_SIZE( all_sets );
        buff->check_space( buff_size );
        PACK_RANGE( buff->buff_ptr, all_sets );
    }
 
    myDebug->tprintf( 4, "Done packing sets.\n" );
 
    buff->set_stored_size();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::unpack_sets( unsigned char*& buff_ptr,
                                     std::vector< EntityHandle >& entities,
                                     const bool store_remote_handles,
                                     const int from_proc )
{
    // Now the sets; assume any sets the application wants to pass are in the entities list
    ErrorCode result;
 
    bool no_sets = ( entities.empty() || ( mbImpl->type_from_handle( *entities.rbegin() ) == MBENTITYSET ) );
 
    Range new_sets;
    int num_sets;
    UNPACK_INT( buff_ptr, num_sets );
 
    if( !num_sets ) return MB_SUCCESS;
 
    int i;
    Range::const_iterator rit;
    std::vector< EntityHandle > members;
    int num_ents;
    std::vector< unsigned int > options_vec( num_sets );
    // Option value
    if( num_sets ) UNPACK_VOID( buff_ptr, &options_vec[0], num_sets * sizeof( unsigned int ) );
 
    // Unpack parallel geometry unique id
    int n_uid;
    UNPACK_INT( buff_ptr, n_uid );
    if( n_uid > 0 && n_uid != num_sets )
    {
        std::cerr << "The number of Parallel geometry unique ids should be same." << std::endl;
    }
 
    if( n_uid > 0 )
    {  // If parallel geometry unique id is packed
        std::vector< int > uids( n_uid );
        UNPACK_INTS( buff_ptr, &uids[0], n_uid );
 
        Tag uid_tag;
        result =
            mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
 
        // Find existing sets
        for( i = 0; i < n_uid; i++ )
        {
            EntityHandle set_handle;
            Range temp_sets;
            void* tag_vals[] = { &uids[i] };
            if( uids[i] > 0 )
            {
                result = mbImpl->get_entities_by_type_and_tag( 0, MBENTITYSET, &uid_tag, tag_vals, 1, temp_sets );
            }
            if( !temp_sets.empty() )
            {  // Existing set
                set_handle = *temp_sets.begin();
            }
            else
            {  // Create a new set
                result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
                result = mbImpl->tag_set_data( uid_tag, &set_handle, 1, &uids[i] );MB_CHK_SET_ERR( result, "Failed to set parallel geometry unique ids" );
            }
            new_sets.insert( set_handle );
        }
    }
    else
    {
        // Create sets
        for( i = 0; i < num_sets; i++ )
        {
            EntityHandle set_handle;
            result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
 
            // Make sure new sets handles are monotonically increasing
            assert( set_handle > *new_sets.rbegin() );
            new_sets.insert( set_handle );
        }
    }
 
    std::copy( new_sets.begin(), new_sets.end(), std::back_inserter( entities ) );
    // Only need to sort if we came in with no sets on the end
    if( !no_sets ) std::sort( entities.begin(), entities.end() );
 
    for( rit = new_sets.begin(), i = 0; rit != new_sets.end(); ++rit, i++ )
    {
        // Unpack entities as vector, with length
        UNPACK_INT( buff_ptr, num_ents );
        members.resize( num_ents );
        if( num_ents ) UNPACK_EH( buff_ptr, &members[0], num_ents );
        result = get_local_handles( &members[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for ordered set contents" );
        result = mbImpl->add_entities( *rit, &members[0], num_ents );MB_CHK_SET_ERR( result, "Failed to add ents to ordered set in unpack" );
    }
 
    std::vector< int > num_pch( 2 * new_sets.size() );
    std::vector< int >::iterator vit;
    int tot_pch = 0;
    for( vit = num_pch.begin(); vit != num_pch.end(); ++vit )
    {
        UNPACK_INT( buff_ptr, *vit );
        tot_pch += *vit;
    }
 
    members.resize( tot_pch );
    UNPACK_EH( buff_ptr, &members[0], tot_pch );
    result = get_local_handles( &members[0], tot_pch, entities );MB_CHK_SET_ERR( result, "Failed to get local handle for parent/child sets" );
 
    int num               = 0;
    EntityHandle* mem_ptr = &members[0];
    for( rit = new_sets.begin(); rit != new_sets.end(); ++rit )
    {
        // Unpack parents/children
        int num_par = num_pch[num++], num_child = num_pch[num++];
        if( num_par + num_child )
        {
            for( i = 0; i < num_par; i++ )
            {
                assert( 0 != mem_ptr[i] );
                result = mbImpl->add_parent_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add parent to set in unpack" );
            }
            mem_ptr += num_par;
            for( i = 0; i < num_child; i++ )
            {
                assert( 0 != mem_ptr[i] );
                result = mbImpl->add_child_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add child to set in unpack" );
            }
            mem_ptr += num_child;
        }
    }
 
    // Unpack source handles
    Range dum_range;
    if( store_remote_handles && !new_sets.empty() )
    {
        UNPACK_RANGE( buff_ptr, dum_range );
        result = update_remote_data( new_sets, dum_range, from_proc, 0 );MB_CHK_SET_ERR( result, "Failed to set sharing data for sets" );
    }
 
    myDebug->tprintf( 4, "Done unpacking sets." );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::pack_adjacencies( Range& /*entities*/,
                                          Range::const_iterator& /*start_rit*/,
                                          Range& /*whole_range*/,
                                          unsigned char*& /*buff_ptr*/,
                                          int& /*count*/,
                                          const bool /*just_count*/,
                                          const bool /*store_handles*/,
                                          const int /*to_proc*/ )
{
    return MB_FAILURE;
}
 
ErrorCode ParallelComm::unpack_adjacencies( unsigned char*& /*buff_ptr*/,
                                            Range& /*entities*/,
                                            const bool /*store_handles*/,
                                            const int /*from_proc*/ )
{
    return MB_FAILURE;
}
 
ErrorCode ParallelComm::pack_tags( Range& entities,
                                   const std::vector< Tag >& src_tags,
                                   const std::vector< Tag >& dst_tags,
                                   const std::vector< Range >& tag_ranges,
                                   Buffer* buff,
                                   const bool store_remote_handles,
                                   const int to_proc )
{
    ErrorCode result;
    std::vector< Tag >::const_iterator tag_it, dst_it;
    std::vector< Range >::const_iterator rit;
    int count = 0;
 
    for( tag_it = src_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); ++tag_it, ++rit )
    {
        result = packed_tag_size( *tag_it, *rit, count );
        if( MB_SUCCESS != result ) return result;
    }
 
    // Number of tags
    count += sizeof( int );
 
    buff->check_space( count );
 
    PACK_INT( buff->buff_ptr, src_tags.size() );
 
    std::vector< EntityHandle > entities_vec( entities.size() );
    std::copy( entities.begin(), entities.end(), entities_vec.begin() );
 
    for( tag_it = src_tags.begin(), dst_it = dst_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end();
         ++tag_it, ++dst_it, ++rit )
    {
        result = pack_tag( *tag_it, *dst_it, *rit, entities_vec, buff, store_remote_handles, to_proc );
        if( MB_SUCCESS != result ) return result;
    }
 
    myDebug->tprintf( 4, "Done packing tags." );
 
    buff->set_stored_size();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::packed_tag_size( Tag tag, const Range& tagged_entities, int& count )
{
    // For dense tags, compute size assuming all entities have that tag
    // For sparse tags, get number of entities w/ that tag to compute size
 
    std::vector< int > var_len_sizes;
    std::vector< const void* > var_len_values;
 
    // Default value
    count += sizeof( int );
    if( NULL != tag->get_default_value() ) count += tag->get_default_value_size();
 
    // Size, type, data type
    count += 3 * sizeof( int );
 
    // Name
    count += sizeof( int );
    count += tag->get_name().size();
 
    // Range of tag
    count += sizeof( int ) + tagged_entities.size() * sizeof( EntityHandle );
 
    if( tag->get_size() == MB_VARIABLE_LENGTH )
    {
        const int num_ent = tagged_entities.size();
        // Send a tag size for each entity
        count += num_ent * sizeof( int );
        // Send tag data for each entity
        var_len_sizes.resize( num_ent );
        var_len_values.resize( num_ent );
        ErrorCode result =
            tag->get_data( sequenceManager, errorHandler, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get lenghts of variable-length tag values" );
        count += std::accumulate( var_len_sizes.begin(), var_len_sizes.end(), 0 );
    }
    else
    {
        // Tag data values for range or vector
        count += tagged_entities.size() * tag->get_size();
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::pack_tag( Tag src_tag,
                                  Tag dst_tag,
                                  const Range& tagged_entities,
                                  const std::vector< EntityHandle >& whole_vec,
                                  Buffer* buff,
                                  const bool store_remote_handles,
                                  const int to_proc )
{
    ErrorCode result;
    std::vector< int > var_len_sizes;
    std::vector< const void* > var_len_values;
 
    if( src_tag != dst_tag )
    {
        if( dst_tag->get_size() != src_tag->get_size() ) return MB_TYPE_OUT_OF_RANGE;
        if( dst_tag->get_data_type() != src_tag->get_data_type() && dst_tag->get_data_type() != MB_TYPE_OPAQUE &&
            src_tag->get_data_type() != MB_TYPE_OPAQUE )
            return MB_TYPE_OUT_OF_RANGE;
    }
 
    // Size, type, data type
    buff->check_space( 3 * sizeof( int ) );
    PACK_INT( buff->buff_ptr, src_tag->get_size() );
    TagType this_type;
    result = mbImpl->tag_get_type( dst_tag, this_type );
    PACK_INT( buff->buff_ptr, (int)this_type );
    DataType data_type = src_tag->get_data_type();
    PACK_INT( buff->buff_ptr, (int)data_type );
    int type_size = TagInfo::size_from_data_type( data_type );
 
    // Default value
    if( NULL == src_tag->get_default_value() )
    {
        buff->check_space( sizeof( int ) );
        PACK_INT( buff->buff_ptr, 0 );
    }
    else
    {
        buff->check_space( src_tag->get_default_value_size() );
        PACK_BYTES( buff->buff_ptr, src_tag->get_default_value(), src_tag->get_default_value_size() );
    }
 
    // Name
    buff->check_space( src_tag->get_name().size() );
    PACK_BYTES( buff->buff_ptr, dst_tag->get_name().c_str(), dst_tag->get_name().size() );
 
    myDebug->tprintf( 4, "Packing tag \"%s\"", src_tag->get_name().c_str() );
    if( src_tag != dst_tag ) myDebug->tprintf( 4, " (as tag \"%s\")", dst_tag->get_name().c_str() );
    myDebug->tprintf( 4, "\n" );
 
    // Pack entities
    buff->check_space( tagged_entities.size() * sizeof( EntityHandle ) + sizeof( int ) );
    PACK_INT( buff->buff_ptr, tagged_entities.size() );
    std::vector< EntityHandle > dum_tagged_entities( tagged_entities.size() );
    result = get_remote_handles( store_remote_handles, tagged_entities, &dum_tagged_entities[0], to_proc, whole_vec );
    if( MB_SUCCESS != result )
    {
        if( myDebug->get_verbosity() == 3 )
        {
            std::cerr << "Failed to get remote handles for tagged entities:" << std::endl;
            tagged_entities.print( "  " );
        }
        MB_SET_ERR( result, "Failed to get remote handles for tagged entities" );
    }
 
    PACK_EH( buff->buff_ptr, &dum_tagged_entities[0], dum_tagged_entities.size() );
 
    const size_t num_ent = tagged_entities.size();
    if( src_tag->get_size() == MB_VARIABLE_LENGTH )
    {
        var_len_sizes.resize( num_ent, 0 );
        var_len_values.resize( num_ent, 0 );
        result = mbImpl->tag_get_by_ptr( src_tag, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get variable-length tag data in pack_tags" );
        buff->check_space( num_ent * sizeof( int ) );
        PACK_INTS( buff->buff_ptr, &var_len_sizes[0], num_ent );
        for( unsigned int i = 0; i < num_ent; i++ )
        {
            buff->check_space( var_len_sizes[i] );
            PACK_VOID( buff->buff_ptr, var_len_values[i], type_size * var_len_sizes[i] );
        }
    }
    else
    {
        buff->check_space( num_ent * src_tag->get_size() );
        // Should be OK to read directly into buffer, since tags are untyped and
        // handled by memcpy
        result = mbImpl->tag_get_data( src_tag, tagged_entities, buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get tag data in pack_tags" );
        buff->buff_ptr += num_ent * src_tag->get_size();
        PC( num_ent * src_tag->get_size(), " void" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_tag_send_list( const Range& whole_range,
                                           std::vector< Tag >& all_tags,
                                           std::vector< Range >& tag_ranges )
{
    std::vector< Tag > tmp_tags;
    ErrorCode result = mbImpl->tag_get_tags( tmp_tags );MB_CHK_SET_ERR( result, "Failed to get tags in pack_tags" );
 
    std::vector< Tag >::iterator tag_it;
    for( tag_it = tmp_tags.begin(); tag_it != tmp_tags.end(); ++tag_it )
    {
        std::string tag_name;
        result = mbImpl->tag_get_name( *tag_it, tag_name );
        if( tag_name.c_str()[0] == '_' && tag_name.c_str()[1] == '_' ) continue;
 
        Range tmp_range;
        result = ( *tag_it )->get_tagged_entities( sequenceManager, tmp_range );MB_CHK_SET_ERR( result, "Failed to get entities for tag in pack_tags" );
        tmp_range = intersect( tmp_range, whole_range );
 
        if( tmp_range.empty() ) continue;
 
        // OK, we'll be sending this tag
        all_tags.push_back( *tag_it );
        tag_ranges.push_back( Range() );
        tag_ranges.back().swap( tmp_range );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::unpack_tags( unsigned char*& buff_ptr,
                                     std::vector< EntityHandle >& entities,
                                     const bool /*store_remote_handles*/,
                                     const int /*from_proc*/,
                                     const MPI_Op* const mpi_op )
{
    // Tags
    // Get all the tags
    // For dense tags, compute size assuming all entities have that tag
    // For sparse tags, get number of entities w/ that tag to compute size
 
    ErrorCode result;
 
    int num_tags;
    UNPACK_INT( buff_ptr, num_tags );
    std::vector< const void* > var_len_vals;
    std::vector< unsigned char > dum_vals;
    std::vector< EntityHandle > dum_ehvals;
 
    for( int i = 0; i < num_tags; i++ )
    {
        // Tag handle
        Tag tag_handle;
 
        // Size, data type
        int tag_size, tag_data_type, tag_type;
        UNPACK_INT( buff_ptr, tag_size );
        UNPACK_INT( buff_ptr, tag_type );
        UNPACK_INT( buff_ptr, tag_data_type );
 
        // Default value
        int def_val_size;
        UNPACK_INT( buff_ptr, def_val_size );
        void* def_val_ptr = NULL;
        if( def_val_size )
        {
            def_val_ptr = buff_ptr;
            buff_ptr += def_val_size;
            UPC( tag_size, " void" );
        }
 
        // Name
        int name_len;
        UNPACK_INT( buff_ptr, name_len );
        std::string tag_name( reinterpret_cast< char* >( buff_ptr ), name_len );
        buff_ptr += name_len;
        UPC( 64, " chars" );
 
        myDebug->tprintf( 4, "Unpacking tag %s\n", tag_name.c_str() );
 
        // Create the tag
        if( tag_size == MB_VARIABLE_LENGTH )
            result = mbImpl->tag_get_handle( tag_name.c_str(), def_val_size, (DataType)tag_data_type, tag_handle,
                                             MB_TAG_VARLEN | MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
        else
            result = mbImpl->tag_get_handle( tag_name.c_str(), tag_size, (DataType)tag_data_type, tag_handle,
                                             MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
        if( MB_SUCCESS != result ) return result;
 
        // Get handles and convert to local handles
        int num_ents;
        UNPACK_INT( buff_ptr, num_ents );
        std::vector< EntityHandle > dum_ents( num_ents );
        UNPACK_EH( buff_ptr, &dum_ents[0], num_ents );
 
        // In this case handles are indices into new entity range; need to convert
        // to local handles
        result = get_local_handles( &dum_ents[0], num_ents, entities );MB_CHK_SET_ERR( result, "Unable to convert to local handles" );
 
        // If it's a handle type, also convert tag vals in-place in buffer
        if( MB_TYPE_HANDLE == tag_type )
        {
            dum_ehvals.resize( num_ents );
            UNPACK_EH( buff_ptr, &dum_ehvals[0], num_ents );
            result = get_local_handles( &dum_ehvals[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for tag vals" );
        }
 
        DataType data_type;
        mbImpl->tag_get_data_type( tag_handle, data_type );
        int type_size = TagInfo::size_from_data_type( data_type );
 
        if( !dum_ents.empty() )
        {
            if( tag_size == MB_VARIABLE_LENGTH )
            {
                // Be careful of alignment here. If the integers are aligned
                // in the buffer, we can use them directly. Otherwise we must
                // copy them.
                std::vector< int > var_lengths( num_ents );
                UNPACK_INTS( buff_ptr, &var_lengths[0], num_ents );
                UPC( sizeof( int ) * num_ents, " void" );
 
                // Get pointers into buffer for each tag value
                var_len_vals.resize( num_ents );
                for( std::vector< EntityHandle >::size_type j = 0; j < (std::vector< EntityHandle >::size_type)num_ents;
                     j++ )
                {
                    var_len_vals[j] = buff_ptr;
                    buff_ptr += var_lengths[j] * type_size;
                    UPC( var_lengths[j], " void" );
                }
                result =
                    mbImpl->tag_set_by_ptr( tag_handle, &dum_ents[0], num_ents, &var_len_vals[0], &var_lengths[0] );MB_CHK_SET_ERR( result, "Failed to set tag data when unpacking variable-length tag" );
            }
            else
            {
                // Get existing values of dst tag
                dum_vals.resize( tag_size * num_ents );
                if( mpi_op )
                {
                    int tag_length;
                    result = mbImpl->tag_get_length( tag_handle, tag_length );MB_CHK_SET_ERR( result, "Failed to get tag length" );
                    result = mbImpl->tag_get_data( tag_handle, &dum_ents[0], num_ents, &dum_vals[0] );MB_CHK_SET_ERR( result, "Failed to get existing value of dst tag on entities" );
                    result = reduce_void( tag_data_type, *mpi_op, tag_length * num_ents, &dum_vals[0], buff_ptr );MB_CHK_SET_ERR( result, "Failed to perform mpi op on dst tags" );
                }
                result = mbImpl->tag_set_data( tag_handle, &dum_ents[0], num_ents, buff_ptr );MB_CHK_SET_ERR( result, "Failed to set range-based tag data when unpacking tag" );
                buff_ptr += num_ents * tag_size;
                UPC( num_ents * tag_size, " void" );
            }
        }
    }
 
    myDebug->tprintf( 4, "Done unpacking tags.\n" );
 
    return MB_SUCCESS;
}
 
template < class T >
T LAND( const T& arg1, const T& arg2 )
{
    return arg1 && arg2;
}
template < class T >
T LOR( const T& arg1, const T& arg2 )
{
    return arg1 || arg2;
}
template < class T >
T LXOR( const T& arg1, const T& arg2 )
{
    return ( ( arg1 && !arg2 ) || ( !arg1 && arg2 ) );
}
template < class T >
T MAX( const T& arg1, const T& arg2 )
{
    return ( arg1 > arg2 ? arg1 : arg2 );
}
template < class T >
T MIN( const T& arg1, const T& arg2 )
{
    return ( arg1 < arg2 ? arg1 : arg2 );
}
template < class T >
T ADD( const T& arg1, const T& arg2 )
{
    return arg1 + arg2;
}
template < class T >
T MULT( const T& arg1, const T& arg2 )
{
    return arg1 * arg2;
}
 
template < class T >
ErrorCode ParallelComm::reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals )
{
    T* old_tmp = reinterpret_cast< T* >( old_vals );
    // T *new_tmp = reinterpret_cast<T*>(new_vals);
    // new vals pointer needs to be aligned , some compilers will optimize and will shift
 
    std::vector< T > new_values;
    new_values.resize( num_ents );
    memcpy( &new_values[0], new_vals, num_ents * sizeof( T ) );
    T* new_tmp = &new_values[0];
 
    if( mpi_op == MPI_SUM )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, ADD< T > );
    else if( mpi_op == MPI_PROD )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MULT< T > );
    else if( mpi_op == MPI_MAX )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MAX< T > );
    else if( mpi_op == MPI_MIN )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MIN< T > );
    else if( mpi_op == MPI_LAND )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LAND< T > );
    else if( mpi_op == MPI_LOR )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LOR< T > );
    else if( mpi_op == MPI_LXOR )
        std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LXOR< T > );
    else if( mpi_op == MPI_BAND || mpi_op == MPI_BOR || mpi_op == MPI_BXOR )
    {
        std::cerr << "Bitwise operations not allowed in tag reductions." << std::endl;
        return MB_FAILURE;
    }
    else if( mpi_op != MPI_OP_NULL )
    {
        std::cerr << "Unknown MPI operation type." << std::endl;
        return MB_TYPE_OUT_OF_RANGE;
    }
 
    // copy now the result back where it should be
    memcpy( new_vals, new_tmp, num_ents * sizeof( T ) );
    std::vector< T >().swap( new_values );  // way to release allocated vector
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::reduce_void( int tag_data_type,
                                     const MPI_Op mpi_op,
                                     int num_ents,
                                     void* old_vals,
                                     void* new_vals )
{
    ErrorCode result;
    switch( tag_data_type )
    {
        case MB_TYPE_INTEGER:
            result = reduce< int >( mpi_op, num_ents, old_vals, new_vals );
            break;
        case MB_TYPE_DOUBLE:
            result = reduce< double >( mpi_op, num_ents, old_vals, new_vals );
            break;
        case MB_TYPE_BIT:
            result = reduce< unsigned char >( mpi_op, num_ents, old_vals, new_vals );
            break;
        default:
            result = MB_SUCCESS;
            break;
    }
 
    return result;
}
 
ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, int resolve_dim, int shared_dim, const Tag* id_tag )
{
    ErrorCode result;
    Range proc_ents;
 
    // Check for structured mesh, and do it differently if it is
    ScdInterface* scdi;
    result = mbImpl->query_interface( scdi );
    if( scdi )
    {
        result = scdi->tag_shared_vertices( this, this_set );
        if( MB_SUCCESS == result )
        {
            myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
            return result;
        }
    }
 
    if( 0 == this_set )
    {
        // Get the entities in the partition sets
        for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit )
        {
            Range tmp_ents;
            result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true );
            if( MB_SUCCESS != result ) return result;
            proc_ents.merge( tmp_ents );
        }
    }
    else
    {
        result = mbImpl->get_entities_by_handle( this_set, proc_ents, true );
        if( MB_SUCCESS != result ) return result;
    }
 
    // Resolve dim is maximal dim of entities in proc_ents
    if( -1 == resolve_dim )
    {
        if( !proc_ents.empty() ) resolve_dim = mbImpl->dimension_from_handle( *proc_ents.rbegin() );
    }
 
    // proc_ents should all be of same dimension
    if( resolve_dim > shared_dim &&
        mbImpl->dimension_from_handle( *proc_ents.rbegin() ) != mbImpl->dimension_from_handle( *proc_ents.begin() ) )
    {
        Range::iterator lower = proc_ents.lower_bound( CN::TypeDimensionMap[0].first ),
                        upper = proc_ents.upper_bound( CN::TypeDimensionMap[resolve_dim - 1].second );
        proc_ents.erase( lower, upper );
    }
 
    // Must call even if we don't have any entities, to make sure
    // collective comm'n works
    return resolve_shared_ents( this_set, proc_ents, resolve_dim, shared_dim, NULL, id_tag );
}
 
ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set,
                                             Range& proc_ents,
                                             int resolve_dim,
                                             int shared_dim,
                                             Range* skin_ents,
                                             const Tag* id_tag )
{
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        define_mpe();
        MPE_Log_event( RESOLVE_START, procConfig.proc_rank(), "Entering resolve_shared_ents." );
    }
#endif
 
    ErrorCode result;
    myDebug->tprintf( 1, "Resolving shared entities.\n" );
 
    if( resolve_dim < shared_dim )
    {
        MB_SET_ERR( MB_FAILURE, "MOAB does not support vertex-based partitions, only element-based ones" );
    }
 
    if( -1 == shared_dim )
    {
        if( !proc_ents.empty() )
            shared_dim = mbImpl->dimension_from_handle( *proc_ents.begin() ) - 1;
        else if( resolve_dim == 3 )
            shared_dim = 2;
    }
    int max_global_resolve_dim = -1;
    int err = MPI_Allreduce( &resolve_dim, &max_global_resolve_dim, 1, MPI_INT, MPI_MAX, proc_config().proc_comm() );
    if( MPI_SUCCESS != err )
    {
        MB_SET_ERR( MB_FAILURE, "Unable to guess global resolve_dim" );
    }
    if( shared_dim < 0 || resolve_dim < 0 )
    {
        // MB_SET_ERR(MB_FAILURE, "Unable to guess shared_dim or resolve_dim");
        resolve_dim = max_global_resolve_dim;
        shared_dim  = resolve_dim - 1;
    }
 
    if( resolve_dim < 0 || shared_dim < 0 ) return MB_SUCCESS;
    // no task has any mesh, get out
 
    // Get the skin entities by dimension
    Range tmp_skin_ents[4];
 
    // Get the entities to be skinned
    // Find the skin
    int skin_dim = resolve_dim - 1;
    if( !skin_ents )
    {
        skin_ents              = tmp_skin_ents;
        skin_ents[resolve_dim] = proc_ents;
        Skinner skinner( mbImpl );
        result =
            skinner.find_skin( this_set, skin_ents[skin_dim + 1], false, skin_ents[skin_dim], NULL, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
        myDebug->tprintf( 1, "Found skin:   skin_dim: %d resolve_dim: %d , now resolving.\n", skin_dim, resolve_dim );
        myDebug->tprintf( 3, "skin_ents[0].size(): %d skin_ents[1].size(): %d  \n", (int)skin_ents[0].size(),
                          (int)skin_ents[1].size() );
        // Get entities adjacent to skin ents from shared_dim down to zero
        for( int this_dim = skin_dim - 1; this_dim >= 0; this_dim-- )
        {
            result =
                mbImpl->get_adjacencies( skin_ents[skin_dim], this_dim, true, skin_ents[this_dim], Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
 
            if( this_set && skin_dim == 2 && this_dim == 1 )
            {
                result = mbImpl->add_entities( this_set, skin_ents[this_dim] );MB_CHK_ERR( result );
            }
        }
    }
    else if( skin_ents[resolve_dim].empty() )
        skin_ents[resolve_dim] = proc_ents;
 
    // Global id tag
    Tag gid_tag;
    if( id_tag )
        gid_tag = *id_tag;
    else
    {
        bool tag_created = false;
        int def_val      = -1;
        result = mbImpl->tag_get_handle( GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE | MB_TAG_CREAT,
                                         &def_val, &tag_created );
        if( MB_ALREADY_ALLOCATED != result && MB_SUCCESS != result )
        {
            MB_SET_ERR( result, "Failed to create/get gid tag handle" );
        }
        else if( tag_created )
        {
            // Just created it, so we need global ids
            result = assign_global_ids( this_set, skin_dim + 1, true, true, true );MB_CHK_SET_ERR( result, "Failed to assign global ids" );
        }
    }
 
    DataType tag_type;
    result = mbImpl->tag_get_data_type( gid_tag, tag_type );MB_CHK_SET_ERR( result, "Failed to get tag data type" );
    int bytes_per_tag;
    result = mbImpl->tag_get_bytes( gid_tag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed to get number of bytes per tag" );
    // On 64 bits, long and int are different
    // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?)
 
    // Get gids for skin ents in a vector, to pass to gs
    std::vector< long > lgid_data( skin_ents[0].size() );
    // Size is either long or int
    // On 64 bit is 8 or 4
    if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
    {  // It is a special id tag
        result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &lgid_data[0] );MB_CHK_SET_ERR( result, "Couldn't get gid tag for skin vertices" );
    }
    else if( 4 == bytes_per_tag )
    {  // Must be GLOBAL_ID tag or 32 bits ...
        std::vector< int > gid_data( lgid_data.size() );
        result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &gid_data[0] );MB_CHK_SET_ERR( result, "Failed to get gid tag for skin vertices" );
        std::copy( gid_data.begin(), gid_data.end(), lgid_data.begin() );
    }
    else
    {
        // Not supported flag
        MB_SET_ERR( MB_FAILURE, "Unsupported id tag" );
    }
 
    // Put handles in vector for passing to gs setup
    std::vector< Ulong > handle_vec;  // Assumes that we can do conversion from Ulong to EntityHandle
    std::copy( skin_ents[0].begin(), skin_ents[0].end(), std::back_inserter( handle_vec ) );
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( SHAREDV_START, procConfig.proc_rank(), "Creating crystal router." );
    }
#endif
 
    // Get a crystal router
    gs_data::crystal_data* cd = procConfig.crystal_router();
 
    /*
    // Get total number of entities; will overshoot highest global id, but
    // that's OK
    int num_total[2] = {0, 0}, num_local[2] = {0, 0};
    result = mbImpl->get_number_entities_by_dimension(this_set, 0, num_local);
    if (MB_SUCCESS != result)return result;
    int failure = MPI_Allreduce(num_local, num_total, 1,
    MPI_INT, MPI_SUM, procConfig.proc_comm());
    if (failure) {
      MB_SET_ERR(MB_FAILURE, "Allreduce for total number of shared ents failed");
    }
    */
    // Call gather-scatter to get shared ids & procs
    gs_data* gsd = new gs_data();
    // assert(sizeof(ulong_) == sizeof(EntityHandle));
    result = gsd->initialize( skin_ents[0].size(), &lgid_data[0], &handle_vec[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
 
    // Get shared proc tags
    Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
    result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
 
    // Load shared verts into a tuple, then sort by index
    TupleList shared_verts;
    shared_verts.initialize( 2, 0, 1, 0, skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ) );
    shared_verts.enableWriteAccess();
 
    unsigned int i = 0, j = 0;
    for( unsigned int p = 0; p < gsd->nlinfo->_np; p++ )
        for( unsigned int np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
        {
            shared_verts.vi_wr[i++] = gsd->nlinfo->_sh_ind[j];
            shared_verts.vi_wr[i++] = gsd->nlinfo->_target[p];
            shared_verts.vul_wr[j]  = gsd->nlinfo->_ulabels[j];
            j++;
            shared_verts.inc_n();
        }
 
    myDebug->tprintf( 3, " shared verts size %d \n", (int)shared_verts.get_n() );
 
    int max_size = skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 );
    moab::TupleList::buffer sort_buffer;
    sort_buffer.buffer_init( max_size );
    shared_verts.sort( 0, &sort_buffer );
    sort_buffer.reset();
 
    // Set sharing procs and handles tags on skin ents
    int maxp = -1;
    std::vector< int > sharing_procs( MAX_SHARING_PROCS );
    std::fill( sharing_procs.begin(), sharing_procs.end(), maxp );
    j = 0;
    i = 0;
 
    // Get ents shared by 1 or n procs
    std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
    Range proc_verts;
    result = mbImpl->get_adjacencies( proc_ents, 0, false, proc_verts, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get proc_verts" );
 
    myDebug->print( 3, " resolve shared ents:  proc verts ", proc_verts );
    result = tag_shared_verts( shared_verts, skin_ents, proc_nvecs, proc_verts );MB_CHK_SET_ERR( result, "Failed to tag shared verts" );
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( SHAREDV_END, procConfig.proc_rank(), "Finished tag_shared_verts." );
    }
#endif
 
    // Get entities shared by 1 or n procs
    result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );MB_CHK_SET_ERR( result, "Failed to tag shared entities" );
 
    shared_verts.reset();
 
    if( myDebug->get_verbosity() > 0 )
    {
        for( std::map< std::vector< int >, std::vector< EntityHandle > >::const_iterator mit = proc_nvecs.begin();
             mit != proc_nvecs.end(); ++mit )
        {
            myDebug->tprintf( 1, "Iface: " );
            for( std::vector< int >::const_iterator vit = ( mit->first ).begin(); vit != ( mit->first ).end(); ++vit )
                myDebug->printf( 1, " %d", *vit );
            myDebug->print( 1, "\n" );
        }
    }
 
    // Create the sets for each interface; store them as tags on
    // the interface instance
    Range iface_sets;
    result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" );
 
    // Establish comm procs and buffers for them
    std::set< unsigned int > procs;
    result = get_interface_procs( procs, true );MB_CHK_SET_ERR( result, "Failed to get interface procs" );
 
#ifndef NDEBUG
    result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Shared handle check failed after interface vertex exchange" );
#endif
 
    // Resolve shared entity remote handles; implemented in ghost cell exchange
    // code because it's so similar
    result = exchange_ghost_cells( -1, -1, 0, 0, true, true );MB_CHK_SET_ERR( result, "Failed to resolve shared entity remote handles" );
 
    // Now build parent/child links for interface sets
    result = create_iface_pc_links();MB_CHK_SET_ERR( result, "Failed to create interface parent/child links" );
 
    gsd->reset();
    delete gsd;
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( RESOLVE_END, procConfig.proc_rank(), "Exiting resolve_shared_ents." );
    }
#endif
 
    // std::ostringstream ent_str;
    // ent_str << "mesh." << procConfig.proc_rank() << ".h5m";
    // mbImpl->write_mesh(ent_str.str().c_str());
 
    // Done
    return result;
}
 
void ParallelComm::define_mpe()
{
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        // Define mpe states used for logging
        int success;
        MPE_Log_get_state_eventIDs( &IFACE_START, &IFACE_END );
        MPE_Log_get_state_eventIDs( &GHOST_START, &GHOST_END );
        MPE_Log_get_state_eventIDs( &SHAREDV_START, &SHAREDV_END );
        MPE_Log_get_state_eventIDs( &RESOLVE_START, &RESOLVE_END );
        MPE_Log_get_state_eventIDs( &ENTITIES_START, &ENTITIES_END );
        MPE_Log_get_state_eventIDs( &RHANDLES_START, &RHANDLES_END );
        MPE_Log_get_state_eventIDs( &OWNED_START, &OWNED_END );
        success = MPE_Describe_state( IFACE_START, IFACE_END, "Resolve interface ents", "green" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( GHOST_START, GHOST_END, "Exchange ghost ents", "red" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( SHAREDV_START, SHAREDV_END, "Resolve interface vertices", "blue" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( RESOLVE_START, RESOLVE_END, "Resolve shared ents", "purple" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( ENTITIES_START, ENTITIES_END, "Exchange shared ents", "yellow" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( RHANDLES_START, RHANDLES_END, "Remote handles", "cyan" );
        assert( MPE_LOG_OK == success );
        success = MPE_Describe_state( OWNED_START, OWNED_END, "Exchange owned ents", "black" );
        assert( MPE_LOG_OK == success );
    }
#endif
}
 
ErrorCode ParallelComm::resolve_shared_ents( ParallelComm** pc,
                                             const unsigned int np,
                                             EntityHandle this_set,
                                             const int part_dim )
{
    std::vector< Range > verts( np );
    int tot_verts = 0;
    unsigned int p, i, j, v;
    ErrorCode rval;
    for( p = 0; p < np; p++ )
    {
        Skinner skinner( pc[p]->get_moab() );
        Range part_ents, skin_ents;
        rval = pc[p]->get_moab()->get_entities_by_dimension( this_set, part_dim, part_ents );
        if( MB_SUCCESS != rval ) return rval;
        rval = skinner.find_skin( this_set, part_ents, false, skin_ents, 0, true, true, true );
        if( MB_SUCCESS != rval ) return rval;
        rval = pc[p]->get_moab()->get_adjacencies( skin_ents, 0, true, verts[p], Interface::UNION );
        if( MB_SUCCESS != rval ) return rval;
        tot_verts += verts[p].size();
    }
 
    TupleList shared_ents;
    shared_ents.initialize( 2, 0, 1, 0, tot_verts );
    shared_ents.enableWriteAccess();
 
    i = 0;
    j = 0;
    std::vector< int > gids;
    Range::iterator rit;
    Tag gid_tag;
    for( p = 0; p < np; p++ )
    {
        gid_tag = pc[p]->get_moab()->globalId_tag();
 
        gids.resize( verts[p].size() );
        rval = pc[p]->get_moab()->tag_get_data( gid_tag, verts[p], &gids[0] );
        if( MB_SUCCESS != rval ) return rval;
 
        for( v = 0, rit = verts[p].begin(); v < gids.size(); v++, ++rit )
        {
            shared_ents.vi_wr[i++] = gids[v];
            shared_ents.vi_wr[i++] = p;
            shared_ents.vul_wr[j]  = *rit;
            j++;
            shared_ents.inc_n();
        }
    }
 
    moab::TupleList::buffer sort_buffer;
    sort_buffer.buffer_init( tot_verts );
    shared_ents.sort( 0, &sort_buffer );
    sort_buffer.reset();
 
    j = 0;
    i = 0;
    std::vector< EntityHandle > handles;
    std::vector< int > procs;
 
    while( i < shared_ents.get_n() )
    {
        handles.clear();
        procs.clear();
 
        // Count & accumulate sharing procs
        int this_gid = shared_ents.vi_rd[j];
        while( i < shared_ents.get_n() && shared_ents.vi_rd[j] == this_gid )
        {
            j++;
            procs.push_back( shared_ents.vi_rd[j++] );
            handles.push_back( shared_ents.vul_rd[i++] );
        }
        if( 1 == procs.size() ) continue;
 
        for( v = 0; v < procs.size(); v++ )
        {
            rval = pc[procs[v]]->update_remote_data( handles[v], &procs[0], &handles[0], procs.size(),
                                                     ( procs[0] == (int)pc[procs[v]]->rank()
                                                           ? PSTATUS_INTERFACE
                                                           : ( PSTATUS_NOT_OWNED | PSTATUS_INTERFACE ) ) );
            if( MB_SUCCESS != rval ) return rval;
        }
    }
 
    std::set< unsigned int > psets;
    for( p = 0; p < np; p++ )
    {
        rval = pc[p]->create_interface_sets( this_set, part_dim, part_dim - 1 );
        if( MB_SUCCESS != rval ) return rval;
        // Establish comm procs and buffers for them
        psets.clear();
        rval = pc[p]->get_interface_procs( psets, true );
        if( MB_SUCCESS != rval ) return rval;
    }
 
    shared_ents.reset();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::tag_iface_entities()
{
    ErrorCode result = MB_SUCCESS;
    Range iface_ents, tmp_ents, rmv_ents;
    std::vector< unsigned char > pstat;
    unsigned char set_pstat;
    Range::iterator rit2;
    unsigned int i;
 
    for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
    {
        iface_ents.clear();
 
        result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get interface set contents" );
        pstat.resize( iface_ents.size() );
        result = mbImpl->tag_get_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set entities" );
        result = mbImpl->tag_get_data( pstatus_tag(), &( *rit ), 1, &set_pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set" );
        rmv_ents.clear();
        for( rit2 = iface_ents.begin(), i = 0; rit2 != iface_ents.end(); ++rit2, i++ )
        {
            if( !( pstat[i] & PSTATUS_INTERFACE ) )
            {
                rmv_ents.insert( *rit2 );
                pstat[i] = 0x0;
            }
        }
        result = mbImpl->remove_entities( *rit, rmv_ents );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
 
        if( !( set_pstat & PSTATUS_NOT_OWNED ) ) continue;
        // If we're here, we need to set the notowned status on (remaining) set contents
 
        // Remove rmv_ents from the contents list
        iface_ents = subtract( iface_ents, rmv_ents );
        // Compress the pstat vector (removing 0x0's)
        std::remove_if( pstat.begin(), pstat.end(),
                        std::bind( std::equal_to< unsigned char >(), std::placeholders::_1, 0x0 ) );
        // std::bind2nd(std::equal_to<unsigned char>(), 0x0));
        // https://stackoverflow.com/questions/32739018/a-replacement-for-stdbind2nd
        // Fold the not_owned bit into remaining values
        unsigned int sz = iface_ents.size();
        for( i = 0; i < sz; i++ )
            pstat[i] |= PSTATUS_NOT_OWNED;
 
        // Set the tag on the entities
        result = mbImpl->tag_set_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus values for interface set entities" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::set_pstatus_entities( Range& pstatus_ents,
                                              unsigned char pstatus_val,
                                              bool lower_dim_ents,
                                              bool verts_too,
                                              int operation )
{
    std::vector< unsigned char > pstatus_vals( pstatus_ents.size() );
    Range all_ents, *range_ptr = &pstatus_ents;
    ErrorCode result;
    if( lower_dim_ents || verts_too )
    {
        all_ents      = pstatus_ents;
        range_ptr     = &all_ents;
        int start_dim = ( lower_dim_ents ? mbImpl->dimension_from_handle( *pstatus_ents.rbegin() ) - 1 : 0 );
        for( ; start_dim >= 0; start_dim-- )
        {
            result = mbImpl->get_adjacencies( all_ents, start_dim, true, all_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get adjacencies for pstatus entities" );
        }
    }
    if( Interface::UNION == operation )
    {
        result = mbImpl->tag_get_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
        for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
            pstatus_vals[i] |= pstatus_val;
    }
    else
    {
        for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
            pstatus_vals[i] = pstatus_val;
    }
    result = mbImpl->tag_set_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::set_pstatus_entities( EntityHandle* pstatus_ents,
                                              int num_ents,
                                              unsigned char pstatus_val,
                                              bool lower_dim_ents,
                                              bool verts_too,
                                              int operation )
{
    std::vector< unsigned char > pstatus_vals( num_ents );
    ErrorCode result;
    if( lower_dim_ents || verts_too )
    {
        // In this case, call the range-based version
        Range tmp_range;
        std::copy( pstatus_ents, pstatus_ents + num_ents, range_inserter( tmp_range ) );
        return set_pstatus_entities( tmp_range, pstatus_val, lower_dim_ents, verts_too, operation );
    }
 
    if( Interface::UNION == operation )
    {
        result = mbImpl->tag_get_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
        for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
            pstatus_vals[i] |= pstatus_val;
    }
    else
    {
        for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
            pstatus_vals[i] = pstatus_val;
    }
    result = mbImpl->tag_set_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
 
    return MB_SUCCESS;
}
 
static size_t choose_owner_idx( const std::vector< unsigned >& proc_list )
{
    // Try to assign owners randomly so we get a good distribution,
    // (note: specifying the same seed on all procs is essential)
    unsigned val = 0;
    for( size_t i = 0; i < proc_list.size(); i++ )
        val ^= proc_list[i];
    srand( (int)( val ) );
    return rand() % proc_list.size();
}
 
struct set_tuple
{
    unsigned idx;
    unsigned proc;
    EntityHandle handle;
    inline bool operator<( set_tuple other ) const
    {
        return ( idx == other.idx ) ? ( proc < other.proc ) : ( idx < other.idx );
    }
};
 
ErrorCode ParallelComm::resolve_shared_sets( EntityHandle file, const Tag* idtag )
{
    // Find all sets with any of the following tags:
    const char* const shared_set_tag_names[] = { GEOM_DIMENSION_TAG_NAME, MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME,
                                                 NEUMANN_SET_TAG_NAME, PARALLEL_PARTITION_TAG_NAME };
    int num_tags                             = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
    Range candidate_sets;
    ErrorCode result = MB_FAILURE;
 
    // If we're not given an ID tag to use to globally identify sets,
    // then fall back to using known tag values
    if( !idtag )
    {
        Tag gid, tag;
        gid = mbImpl->globalId_tag();
        if( NULL != gid ) result = mbImpl->tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tag );
        if( MB_SUCCESS == result )
        {
            for( int d = 0; d < 4; d++ )
            {
                candidate_sets.clear();
                const void* vals[] = { &d };
                result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, vals, 1, candidate_sets );
                if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, gid );
            }
        }
 
        for( int i = 1; i < num_tags; i++ )
        {
            result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag );
            if( MB_SUCCESS == result )
            {
                candidate_sets.clear();
                result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets );
                if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, tag );
            }
        }
 
        return MB_SUCCESS;
    }
 
    for( int i = 0; i < num_tags; i++ )
    {
        Tag tag;
        result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag, MB_TAG_ANY );
        if( MB_SUCCESS != result ) continue;
 
        mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets, Interface::UNION );
    }
 
    // Find any additional sets that contain shared entities
    Range::iterator hint = candidate_sets.begin();
    Range all_sets;
    mbImpl->get_entities_by_type( file, MBENTITYSET, all_sets );
    all_sets           = subtract( all_sets, candidate_sets );
    Range::iterator it = all_sets.begin();
    while( it != all_sets.end() )
    {
        Range contents;
        mbImpl->get_entities_by_handle( *it, contents );
        contents.erase( contents.lower_bound( MBENTITYSET ), contents.end() );
        filter_pstatus( contents, PSTATUS_SHARED, PSTATUS_OR );
        if( contents.empty() )
        {
            ++it;
        }
        else
        {
            hint = candidate_sets.insert( hint, *it );
            it   = all_sets.erase( it );
        }
    }
 
    // Find any additionl sets that contain or are parents of potential shared sets
    Range prev_list = candidate_sets;
    while( !prev_list.empty() )
    {
        it = all_sets.begin();
        Range new_list;
        hint = new_list.begin();
        while( it != all_sets.end() )
        {
            Range contents;
            mbImpl->get_entities_by_type( *it, MBENTITYSET, contents );
            if( !intersect( prev_list, contents ).empty() )
            {
                hint = new_list.insert( hint, *it );
                it   = all_sets.erase( it );
            }
            else
            {
                new_list.clear();
                mbImpl->get_child_meshsets( *it, contents );
                if( !intersect( prev_list, contents ).empty() )
                {
                    hint = new_list.insert( hint, *it );
                    it   = all_sets.erase( it );
                }
                else
                {
                    ++it;
                }
            }
        }
 
        candidate_sets.merge( new_list );
        prev_list.swap( new_list );
    }
 
    return resolve_shared_sets( candidate_sets, *idtag );
}
 
#ifndef NDEBUG
bool is_sorted_unique( std::vector< unsigned >& v )
{
    for( size_t i = 1; i < v.size(); i++ )
        if( v[i - 1] >= v[i] ) return false;
    return true;
}
#endif
 
ErrorCode ParallelComm::resolve_shared_sets( Range& sets, Tag idtag )
{
    ErrorCode result;
    const unsigned rk = proc_config().proc_rank();
    MPI_Comm cm       = proc_config().proc_comm();
 
    // Build sharing list for all sets
 
    // Get ids for sets in a vector, to pass to gs
    std::vector< long > larray;  // Allocate sufficient space for longs
    std::vector< Ulong > handles;
    Range tmp_sets;
    // The id tag can be size 4 or size 8
    // Based on that, convert to int or to long, similarly to what we do
    // for resolving shared vertices;
    // This code must work on 32 bit too, where long is 4 bytes, also
    // so test first size 4, then we should be fine
    DataType tag_type;
    result = mbImpl->tag_get_data_type( idtag, tag_type );MB_CHK_SET_ERR( result, "Failed getting tag data type" );
    int bytes_per_tag;
    result = mbImpl->tag_get_bytes( idtag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed getting number of bytes per tag" );
    // On 64 bits, long and int are different
    // On 32 bits, they are not; if size of long is 8, it is a 64 bit machine (really?)
 
    for( Range::iterator rit = sets.begin(); rit != sets.end(); ++rit )
    {
        if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
        {  // It is a special id tag
            long dum;
            result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
            if( MB_SUCCESS == result )
            {
                larray.push_back( dum );
                handles.push_back( *rit );
                tmp_sets.insert( tmp_sets.end(), *rit );
            }
        }
        else if( 4 == bytes_per_tag )
        {  // Must be GLOBAL_ID tag or MATERIAL_ID, etc
            int dum;
            result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
            if( MB_SUCCESS == result )
            {
                larray.push_back( dum );
                handles.push_back( *rit );
                tmp_sets.insert( tmp_sets.end(), *rit );
            }
        }
    }
 
    const size_t nsets = handles.size();
 
    // Get handle array for sets
    // This is not true on windows machine, 64 bits: entity handle is 64 bit, long is 32
    // assert(sizeof(EntityHandle) <= sizeof(unsigned long));
 
    // Do communication of data
    gs_data::crystal_data* cd = procConfig.crystal_router();
    gs_data* gsd              = new gs_data();
    result                    = gsd->initialize( nsets, &larray[0], &handles[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
 
    // Convert from global IDs grouped by process rank to list
    // of <idx, rank> pairs so that we can sort primarily
    // by idx and secondarily by rank (we want lists of procs for each
    // idx, not lists if indices for each proc).
    size_t ntuple = 0;
    for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
        ntuple += gsd->nlinfo->_nshared[p];
    std::vector< set_tuple > tuples;
    tuples.reserve( ntuple );
    size_t j = 0;
    for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
    {
        for( unsigned np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
        {
            set_tuple t;
            t.idx    = gsd->nlinfo->_sh_ind[j];
            t.proc   = gsd->nlinfo->_target[p];
            t.handle = gsd->nlinfo->_ulabels[j];
            tuples.push_back( t );
            j++;
        }
    }
    std::sort( tuples.begin(), tuples.end() );
 
    // Release crystal router stuff
    gsd->reset();
    delete gsd;
 
    // Storing sharing data for each set
    size_t ti    = 0;
    unsigned idx = 0;
    std::vector< unsigned > procs;
    Range::iterator si = tmp_sets.begin();
    while( si != tmp_sets.end() && ti < tuples.size() )
    {
        assert( idx <= tuples[ti].idx );
        if( idx < tuples[ti].idx ) si += ( tuples[ti].idx - idx );
        idx = tuples[ti].idx;
 
        procs.clear();
        size_t ti_init = ti;
        while( ti < tuples.size() && tuples[ti].idx == idx )
        {
            procs.push_back( tuples[ti].proc );
            ++ti;
        }
        assert( is_sorted_unique( procs ) );
 
        result = sharedSetData->set_sharing_procs( *si, procs );
        if( MB_SUCCESS != result )
        {
            std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
            std::cerr.flush();
            MPI_Abort( cm, 1 );
        }
 
        // Add this proc to list of sharing procs in correct position
        // so that all procs select owner based on same list
        std::vector< unsigned >::iterator it = std::lower_bound( procs.begin(), procs.end(), rk );
        assert( it == procs.end() || *it > rk );
        procs.insert( it, rk );
        size_t owner_idx = choose_owner_idx( procs );
        EntityHandle owner_handle;
        if( procs[owner_idx] == rk )
            owner_handle = *si;
        else if( procs[owner_idx] > rk )
            owner_handle = tuples[ti_init + owner_idx - 1].handle;
        else
            owner_handle = tuples[ti_init + owner_idx].handle;
        result = sharedSetData->set_owner( *si, procs[owner_idx], owner_handle );
        if( MB_SUCCESS != result )
        {
            std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
            std::cerr.flush();
            MPI_Abort( cm, 1 );
        }
 
        ++si;
        ++idx;
    }
 
    return MB_SUCCESS;
}
// populate sets with ghost entities, if necessary
ErrorCode ParallelComm::augment_default_sets_with_ghosts( EntityHandle file_set )
{
    // gather all default sets we are interested in, material, neumann, etc
    // we will skip geometry sets, because they are not uniquely identified with their tag value
    // maybe we will add another tag, like category
 
    if( procConfig.proc_size() < 2 ) return MB_SUCCESS;  // no reason to stop by
    const char* const shared_set_tag_names[] = { MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, NEUMANN_SET_TAG_NAME,
                                                 PARALLEL_PARTITION_TAG_NAME };
 
    int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
 
    Range* rangeSets = new Range[num_tags];
    Tag* tags        = new Tag[num_tags + 1];  // one extra for global id tag, which is an int, so far
 
    int my_rank   = rank();
    int** tagVals = new int*[num_tags];
    for( int i = 0; i < num_tags; i++ )
        tagVals[i] = NULL;
    ErrorCode rval;
 
    // for each tag, we keep a local map, from the value to the actual set with that value
    // we assume that the tag values are unique, for a given set, otherwise we
    // do not know to which set to add the entity
 
    typedef std::map< int, EntityHandle > MVal;
    typedef std::map< int, EntityHandle >::iterator itMVal;
    MVal* localMaps = new MVal[num_tags];
 
    for( int i = 0; i < num_tags; i++ )
    {
 
        rval = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tags[i], MB_TAG_ANY );
        if( MB_SUCCESS != rval ) continue;
        rval = mbImpl->get_entities_by_type_and_tag( file_set, MBENTITYSET, &( tags[i] ), 0, 1, rangeSets[i],
                                                     Interface::UNION );MB_CHK_SET_ERR( rval, "can't get sets with a tag" );
 
        if( rangeSets[i].size() > 0 )
        {
            tagVals[i] = new int[rangeSets[i].size()];
            // fill up with the tag values
            rval = mbImpl->tag_get_data( tags[i], rangeSets[i], tagVals[i] );MB_CHK_SET_ERR( rval, "can't get set tag values" );
            // now for inverse mapping:
            for( int j = 0; j < (int)rangeSets[i].size(); j++ )
            {
                localMaps[i][tagVals[i][j]] = rangeSets[i][j];
            }
        }
    }
    // get the global id tag too
    tags[num_tags] = mbImpl->globalId_tag();
 
    TupleList remoteEnts;
    // processor to send to, type of tag (0-mat,) tag value,     remote handle
    //                         1-diri
    //                         2-neum
    //                         3-part
    //
    int initialSize = (int)sharedEnts.size();  // estimate that on average, each shared ent
    // will be sent to one processor, for one tag
    // we will actually send only entities that are owned locally, and from those
    // only those that do have a special tag (material, neumann, etc)
    // if we exceed the capacity, we resize the tuple
    remoteEnts.initialize( 3, 0, 1, 0, initialSize );
    remoteEnts.enableWriteAccess();
 
    // now, for each owned entity, get the remote handle(s) and Proc(s), and verify if it
    // belongs to one of the sets; if yes, create a tuple and append it
 
    std::set< EntityHandle > own_and_sha;
    int ir = 0, jr = 0;
    for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
    {
        // ghosted eh
        EntityHandle geh = *vit;
        if( own_and_sha.find( geh ) != own_and_sha.end() )  // already encountered
            continue;
        int procs[MAX_SHARING_PROCS];
        EntityHandle handles[MAX_SHARING_PROCS];
        int nprocs;
        unsigned char pstat;
        rval = get_sharing_data( geh, procs, handles, pstat, nprocs );
        if( rval != MB_SUCCESS )
        {
            for( int i = 0; i < num_tags; i++ )
                delete[] tagVals[i];
            delete[] tagVals;
 
            MB_CHK_SET_ERR( rval, "Failed to get sharing data" );
        }
        if( pstat & PSTATUS_NOT_OWNED ) continue;  // we will send info only for entities that we own
        own_and_sha.insert( geh );
        for( int i = 0; i < num_tags; i++ )
        {
            for( int j = 0; j < (int)rangeSets[i].size(); j++ )
            {
                EntityHandle specialSet = rangeSets[i][j];  // this set has tag i, value tagVals[i][j];
                if( mbImpl->contains_entities( specialSet, &geh, 1 ) )
                {
                    // this ghosted entity is in a special set, so form the tuple
                    // to send to the processors that do not own this
                    for( int k = 0; k < nprocs; k++ )
                    {
                        if( procs[k] != my_rank )
                        {
                            if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
                            {
                                // resize, so we do not overflow
                                int oldSize = remoteEnts.get_max();
                                // increase with 50% the capacity
                                remoteEnts.resize( oldSize + oldSize / 2 + 1 );
                            }
                            remoteEnts.vi_wr[ir++]  = procs[k];       // send to proc
                            remoteEnts.vi_wr[ir++]  = i;              // for the tags [i] (0-3)
                            remoteEnts.vi_wr[ir++]  = tagVals[i][j];  // actual value of the tag
                            remoteEnts.vul_wr[jr++] = handles[k];
                            remoteEnts.inc_n();
                        }
                    }
                }
            }
        }
        // if the local entity has a global id, send it too, so we avoid
        // another "exchange_tags" for global id
        int gid;
        rval = mbImpl->tag_get_data( tags[num_tags], &geh, 1, &gid );MB_CHK_SET_ERR( rval, "Failed to get global id" );
        if( gid != 0 )
        {
            for( int k = 0; k < nprocs; k++ )
            {
                if( procs[k] != my_rank )
                {
                    if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
                    {
                        // resize, so we do not overflow
                        int oldSize = remoteEnts.get_max();
                        // increase with 50% the capacity
                        remoteEnts.resize( oldSize + oldSize / 2 + 1 );
                    }
                    remoteEnts.vi_wr[ir++]  = procs[k];  // send to proc
                    remoteEnts.vi_wr[ir++]  = num_tags;  // for the tags [j] (4)
                    remoteEnts.vi_wr[ir++]  = gid;       // actual value of the tag
                    remoteEnts.vul_wr[jr++] = handles[k];
                    remoteEnts.inc_n();
                }
            }
        }
    }
 
#ifndef NDEBUG
    if( my_rank == 1 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 1, before augment routing" );
    MPI_Barrier( procConfig.proc_comm() );
    int sentEnts = remoteEnts.get_n();
    assert( ( sentEnts == jr ) && ( 3 * sentEnts == ir ) );
#endif
    // exchange the info now, and send to
    gs_data::crystal_data* cd = this->procConfig.crystal_router();
    // All communication happens here; no other mpi calls
    // Also, this is a collective call
    rval = cd->gs_transfer( 1, remoteEnts, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" );
#ifndef NDEBUG
    if( my_rank == 0 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 0, after augment routing" );
    MPI_Barrier( procConfig.proc_comm() );
#endif
 
    // now process the data received from other processors
    int received = remoteEnts.get_n();
    for( int i = 0; i < received; i++ )
    {
        // int from = ents_to_delete.vi_rd[i];
        EntityHandle geh = (EntityHandle)remoteEnts.vul_rd[i];
        int from_proc    = remoteEnts.vi_rd[3 * i];
        if( my_rank == from_proc )
            std::cout << " unexpected receive from my rank " << my_rank << " during augmenting with ghosts\n ";
        int tag_type = remoteEnts.vi_rd[3 * i + 1];
        assert( ( 0 <= tag_type ) && ( tag_type <= num_tags ) );
        int value = remoteEnts.vi_rd[3 * i + 2];
        if( tag_type == num_tags )
        {
            // it is global id
            rval = mbImpl->tag_set_data( tags[num_tags], &geh, 1, &value );MB_CHK_SET_ERR( rval, "Error in setting gid tag" );
        }
        else
        {
            // now, based on value and tag type, see if we have that value in the map
            MVal& lmap = localMaps[tag_type];
            itMVal itm = lmap.find( value );
            if( itm == lmap.end() )
            {
                // the value was not found yet in the local map, so we have to create the set
                EntityHandle newSet;
                rval = mbImpl->create_meshset( MESHSET_SET, newSet );MB_CHK_SET_ERR( rval, "can't create new set" );
                lmap[value] = newSet;
                // set the tag value
                rval = mbImpl->tag_set_data( tags[tag_type], &newSet, 1, &value );MB_CHK_SET_ERR( rval, "can't set tag for new set" );
 
                // we also need to add the new created set to the file set, if not null
                if( file_set )
                {
                    rval = mbImpl->add_entities( file_set, &newSet, 1 );MB_CHK_SET_ERR( rval, "can't add new set to the file set" );
                }
            }
            // add the entity to the set pointed to by the map
            rval = mbImpl->add_entities( lmap[value], &geh, 1 );MB_CHK_SET_ERR( rval, "can't add ghost ent to the set" );
        }
    }
 
    for( int i = 0; i < num_tags; i++ )
        delete[] tagVals[i];
    delete[] tagVals;
    delete[] rangeSets;
    delete[] tags;
    delete[] localMaps;
    return MB_SUCCESS;
}
ErrorCode ParallelComm::create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim )
{
    std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
 
    // Build up the list of shared entities
    int procs[MAX_SHARING_PROCS];
    EntityHandle handles[MAX_SHARING_PROCS];
    ErrorCode result;
    int nprocs;
    unsigned char pstat;
    for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
    {
        if( shared_dim != -1 && mbImpl->dimension_from_handle( *vit ) > shared_dim ) continue;
        result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
        std::sort( procs, procs + nprocs );
        std::vector< int > tmp_procs( procs, procs + nprocs );
        assert( tmp_procs.size() != 2 );
        proc_nvecs[tmp_procs].push_back( *vit );
    }
 
    Skinner skinner( mbImpl );
    Range skin_ents[4];
    result = mbImpl->get_entities_by_dimension( this_set, resolve_dim, skin_ents[resolve_dim] );MB_CHK_SET_ERR( result, "Failed to get skin entities by dimension" );
    result =
        skinner.find_skin( this_set, skin_ents[resolve_dim], false, skin_ents[resolve_dim - 1], 0, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
    if( shared_dim > 1 )
    {
        result = mbImpl->get_adjacencies( skin_ents[resolve_dim - 1], resolve_dim - 2, true, skin_ents[resolve_dim - 2],
                                          Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
    }
 
    result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );
 
    return create_interface_sets( proc_nvecs );
}
 
ErrorCode ParallelComm::create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
{
    if( proc_nvecs.empty() ) return MB_SUCCESS;
 
    int proc_ids[MAX_SHARING_PROCS];
    EntityHandle proc_handles[MAX_SHARING_PROCS];
    Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
    ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in create_interface_sets" );
    Range::iterator rit;
 
    // Create interface sets, tag them, and tag their contents with iface set tag
    std::vector< unsigned char > pstatus;
    for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator vit = proc_nvecs.begin();
         vit != proc_nvecs.end(); ++vit )
    {
        // Create the set
        EntityHandle new_set;
        result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
        interfaceSets.insert( new_set );
 
        // Add entities
        assert( !vit->second.empty() );
        result = mbImpl->add_entities( new_set, &( vit->second )[0], ( vit->second ).size() );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
        // Tag set with the proc rank(s)
        if( vit->first.size() == 1 )
        {
            assert( ( vit->first )[0] != (int)procConfig.proc_rank() );
            result = mbImpl->tag_set_data( shp_tag, &new_set, 1, &( vit->first )[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            proc_handles[0] = 0;
            result          = mbImpl->tag_set_data( shh_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
        }
        else
        {
            // Pad tag data out to MAX_SHARING_PROCS with -1
            if( vit->first.size() > MAX_SHARING_PROCS )
            {
                std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_set ) )
                          << ' ' << ID_FROM_HANDLE( new_set ) << " on process " << proc_config().proc_rank()
                          << std::endl;
                std::cerr.flush();
                MPI_Abort( proc_config().proc_comm(), 66 );
            }
            // assert(vit->first.size() <= MAX_SHARING_PROCS);
            std::copy( vit->first.begin(), vit->first.end(), proc_ids );
            std::fill( proc_ids + vit->first.size(), proc_ids + MAX_SHARING_PROCS, -1 );
            result = mbImpl->tag_set_data( shps_tag, &new_set, 1, proc_ids );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            unsigned int ind = std::find( proc_ids, proc_ids + vit->first.size(), procConfig.proc_rank() ) - proc_ids;
            assert( ind < vit->first.size() );
            std::fill( proc_handles, proc_handles + MAX_SHARING_PROCS, 0 );
            proc_handles[ind] = new_set;
            result            = mbImpl->tag_set_data( shhs_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
        }
 
        // Get the owning proc, then set the pstatus tag on iface set
        int min_proc       = ( vit->first )[0];
        unsigned char pval = ( PSTATUS_SHARED | PSTATUS_INTERFACE );
        if( min_proc < (int)procConfig.proc_rank() ) pval |= PSTATUS_NOT_OWNED;
        if( vit->first.size() > 1 ) pval |= PSTATUS_MULTISHARED;
        result = mbImpl->tag_set_data( pstat_tag, &new_set, 1, &pval );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
 
        // Tag the vertices with the same thing
        pstatus.clear();
        std::vector< EntityHandle > verts;
        for( std::vector< EntityHandle >::iterator v2it = ( vit->second ).begin(); v2it != ( vit->second ).end();
             ++v2it )
            if( mbImpl->type_from_handle( *v2it ) == MBVERTEX ) verts.push_back( *v2it );
        pstatus.resize( verts.size(), pval );
        if( !verts.empty() )
        {
            result = mbImpl->tag_set_data( pstat_tag, &verts[0], verts.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set vertices with pstatus" );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::create_iface_pc_links()
{
    // Now that we've resolved the entities in the iface sets,
    // set parent/child links between the iface sets
 
    // First tag all entities in the iface sets
    Tag tmp_iface_tag;
    EntityHandle tmp_iface_set = 0;
    ErrorCode result           = mbImpl->tag_get_handle( "__tmp_iface", 1, MB_TYPE_HANDLE, tmp_iface_tag,
                                                         MB_TAG_DENSE | MB_TAG_CREAT, &tmp_iface_set );MB_CHK_SET_ERR( result, "Failed to create temporary interface set tag" );
 
    Range iface_ents;
    std::vector< EntityHandle > tag_vals;
    Range::iterator rit;
 
    for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
    {
        // tag entities with interface set
        iface_ents.clear();
        result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in interface set" );
 
        if( iface_ents.empty() ) continue;
 
        tag_vals.resize( iface_ents.size() );
        std::fill( tag_vals.begin(), tag_vals.end(), *rit );
        result = mbImpl->tag_set_data( tmp_iface_tag, iface_ents, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to tag iface entities with interface set" );
    }
 
    // Now go back through interface sets and add parent/child links
    Range tmp_ents2;
    for( int d = 2; d >= 0; d-- )
    {
        for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
        {
            // Get entities on this interface
            iface_ents.clear();
            result = mbImpl->get_entities_by_handle( *rit, iface_ents, true );MB_CHK_SET_ERR( result, "Failed to get entities by handle" );
            if( iface_ents.empty() || mbImpl->dimension_from_handle( *iface_ents.rbegin() ) != d ) continue;
 
            // Get higher-dimensional entities and their interface sets
            result = mbImpl->get_adjacencies( &( *iface_ents.begin() ), 1, d + 1, false, tmp_ents2 );MB_CHK_SET_ERR( result, "Failed to get adjacencies for interface sets" );
            tag_vals.resize( tmp_ents2.size() );
            result = mbImpl->tag_get_data( tmp_iface_tag, tmp_ents2, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to get tmp iface tag for interface sets" );
 
            // Go through and for any on interface make it a parent
            EntityHandle last_set = 0;
            for( unsigned int i = 0; i < tag_vals.size(); i++ )
            {
                if( tag_vals[i] && tag_vals[i] != last_set )
                {
                    result = mbImpl->add_parent_child( tag_vals[i], *rit );MB_CHK_SET_ERR( result, "Failed to add parent/child link for interface set" );
                    last_set = tag_vals[i];
                }
            }
        }
    }
 
    // Delete the temporary tag
    result = mbImpl->tag_delete( tmp_iface_tag );MB_CHK_SET_ERR( result, "Failed to delete tmp iface tag" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_proc_nvecs( int resolve_dim,
                                        int shared_dim,
                                        Range* skin_ents,
                                        std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
{
    // Set sharing procs tags on other skin ents
    ErrorCode result;
    const EntityHandle* connect;
    int num_connect;
    std::set< int > sharing_procs;
    std::vector< EntityHandle > dum_connect;
    std::vector< int > sp_vec;
 
    for( int d = 3; d > 0; d-- )
    {
        if( resolve_dim == d ) continue;
 
        for( Range::iterator rit = skin_ents[d].begin(); rit != skin_ents[d].end(); ++rit )
        {
            // Get connectivity
            result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect );MB_CHK_SET_ERR( result, "Failed to get connectivity on non-vertex skin entities" );
 
            int op = ( resolve_dim < shared_dim ? Interface::UNION : Interface::INTERSECT );
            result = get_sharing_data( connect, num_connect, sharing_procs, op );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_proc_nvecs" );
            if( sharing_procs.empty() ||
                ( sharing_procs.size() == 1 && *sharing_procs.begin() == (int)procConfig.proc_rank() ) )
                continue;
 
            // Need to specify sharing data correctly for entities or they will
            // end up in a different interface set than corresponding vertices
            if( sharing_procs.size() == 2 )
            {
                std::set< int >::iterator it = sharing_procs.find( proc_config().proc_rank() );
                assert( it != sharing_procs.end() );
                sharing_procs.erase( it );
            }
 
            // Intersection is the owning proc(s) for this skin ent
            sp_vec.clear();
            std::copy( sharing_procs.begin(), sharing_procs.end(), std::back_inserter( sp_vec ) );
            assert( sp_vec.size() != 2 );
            proc_nvecs[sp_vec].push_back( *rit );
        }
    }
 
#ifndef NDEBUG
    // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
    for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
         mit != proc_nvecs.end(); ++mit )
    {
        std::vector< EntityHandle > tmp_vec = ( mit->second );
        std::sort( tmp_vec.begin(), tmp_vec.end() );
        std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
        assert( vit == tmp_vec.end() );
    }
#endif
 
    return MB_SUCCESS;
}
 
// Overloaded form of tag_shared_verts
// Tuple coming in is of form (arbitrary value, remoteProc, localHandle, remoteHandle)
// Also will check for doubles in the list if the list is sorted
ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
                                          std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
                                          Range& /*proc_verts*/,
                                          unsigned int i_extra )
{
    Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
    ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
 
    unsigned int j = 0, i = 0;
    std::vector< int > sharing_procs, sharing_procs2, tag_procs;
    std::vector< EntityHandle > sharing_handles, sharing_handles2, tag_lhandles, tag_rhandles;
    std::vector< unsigned char > pstatus;
 
    // Were on tuple j/2
    if( i_extra ) i += i_extra;
    while( j < 2 * shared_ents.get_n() )
    {
        // Count & accumulate sharing procs
        EntityHandle this_ent = shared_ents.vul_rd[j], other_ent = 0;
        int other_proc = -1;
        while( j < 2 * shared_ents.get_n() && shared_ents.vul_rd[j] == this_ent )
        {
            j++;
            // Shouldn't have same proc
            assert( shared_ents.vi_rd[i] != (int)procConfig.proc_rank() );
            // Grab the remote data if its not a dublicate
            if( shared_ents.vul_rd[j] != other_ent || shared_ents.vi_rd[i] != other_proc )
            {
                assert( 0 != shared_ents.vul_rd[j] );
                sharing_procs.push_back( shared_ents.vi_rd[i] );
                sharing_handles.push_back( shared_ents.vul_rd[j] );
            }
            other_proc = shared_ents.vi_rd[i];
            other_ent  = shared_ents.vul_rd[j];
            j++;
            i += 1 + i_extra;
        }
 
        if( sharing_procs.size() > 1 )
        {
            // Add current proc/handle to list
            sharing_procs.push_back( procConfig.proc_rank() );
            sharing_handles.push_back( this_ent );
 
            // Sort sharing_procs and sharing_handles such that
            // sharing_procs is in ascending order. Use temporary
            // lists and binary search to re-order sharing_handles.
            sharing_procs2 = sharing_procs;
            std::sort( sharing_procs2.begin(), sharing_procs2.end() );
            sharing_handles2.resize( sharing_handles.size() );
            for( size_t k = 0; k < sharing_handles.size(); k++ )
            {
                size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
                             sharing_procs2.begin();
                sharing_handles2[idx] = sharing_handles[k];
            }
            sharing_procs.swap( sharing_procs2 );
            sharing_handles.swap( sharing_handles2 );
        }
 
        assert( sharing_procs.size() != 2 );
        proc_nvecs[sharing_procs].push_back( this_ent );
 
        unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
        if( sharing_procs.size() == 1 )
        {
            tag_procs.push_back( sharing_procs[0] );
            tag_lhandles.push_back( this_ent );
            tag_rhandles.push_back( sharing_handles[0] );
            pstatus.push_back( share_flag );
        }
        else
        {
            // Pad lists
            // assert(sharing_procs.size() <= MAX_SHARING_PROCS);
            if( sharing_procs.size() > MAX_SHARING_PROCS )
            {
                std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
                          << proc_config().proc_rank() << std::endl;
                std::cerr.flush();
                MPI_Abort( proc_config().proc_comm(), 66 );
            }
            sharing_procs.resize( MAX_SHARING_PROCS, -1 );
            sharing_handles.resize( MAX_SHARING_PROCS, 0 );
            result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
            result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
            result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
            sharedEnts.insert( this_ent );
        }
 
        // Reset sharing proc(s) tags
        sharing_procs.clear();
        sharing_handles.clear();
    }
 
    if( !tag_procs.empty() )
    {
        result = mbImpl->tag_set_data( shp_tag, &tag_lhandles[0], tag_procs.size(), &tag_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
        result = mbImpl->tag_set_data( shh_tag, &tag_lhandles[0], tag_procs.size(), &tag_rhandles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
        result = mbImpl->tag_set_data( pstat_tag, &tag_lhandles[0], tag_procs.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
        for( std::vector< EntityHandle >::iterator vvt = tag_lhandles.begin(); vvt != tag_lhandles.end(); vvt++ )
            sharedEnts.insert( *vvt );
    }
 
#ifndef NDEBUG
    // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
    for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
         mit != proc_nvecs.end(); ++mit )
    {
        std::vector< EntityHandle > tmp_vec = ( mit->second );
        std::sort( tmp_vec.begin(), tmp_vec.end() );
        std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
        assert( vit == tmp_vec.end() );
    }
#endif
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
                                          Range* skin_ents,
                                          std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
                                          Range& /*proc_verts*/ )
{
    Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
    ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
 
    unsigned int j = 0, i = 0;
    std::vector< int > sharing_procs, sharing_procs2;
    std::vector< EntityHandle > sharing_handles, sharing_handles2, skin_verts( skin_ents[0].size() );
    for( Range::iterator rit = skin_ents[0].begin(); rit != skin_ents[0].end(); ++rit, i++ )
        skin_verts[i] = *rit;
    i = 0;
 
    while( j < 2 * shared_ents.get_n() )
    {
        // Count & accumulate sharing procs
        int this_idx          = shared_ents.vi_rd[j];
        EntityHandle this_ent = skin_verts[this_idx];
        while( j < 2 * shared_ents.get_n() && shared_ents.vi_rd[j] == this_idx )
        {
            j++;
            // Shouldn't have same proc
            assert( shared_ents.vi_rd[j] != (int)procConfig.proc_rank() );
            sharing_procs.push_back( shared_ents.vi_rd[j++] );
            sharing_handles.push_back( shared_ents.vul_rd[i++] );
        }
 
        if( sharing_procs.size() > 1 )
        {
            // Add current proc/handle to list
            sharing_procs.push_back( procConfig.proc_rank() );
            sharing_handles.push_back( this_ent );
        }
 
        // Sort sharing_procs and sharing_handles such that
        // sharing_procs is in ascending order. Use temporary
        // lists and binary search to re-order sharing_handles.
        sharing_procs2 = sharing_procs;
        std::sort( sharing_procs2.begin(), sharing_procs2.end() );
        sharing_handles2.resize( sharing_handles.size() );
        for( size_t k = 0; k < sharing_handles.size(); k++ )
        {
            size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
                         sharing_procs2.begin();
            sharing_handles2[idx] = sharing_handles[k];
        }
        sharing_procs.swap( sharing_procs2 );
        sharing_handles.swap( sharing_handles2 );
 
        assert( sharing_procs.size() != 2 );
        proc_nvecs[sharing_procs].push_back( this_ent );
 
        unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
        if( sharing_procs.size() == 1 )
        {
            result = mbImpl->tag_set_data( shp_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
            result = mbImpl->tag_set_data( shh_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
            result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &share_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
            sharedEnts.insert( this_ent );
        }
        else
        {
            // Pad lists
            // assert(sharing_procs.size() <= MAX_SHARING_PROCS);
            if( sharing_procs.size() > MAX_SHARING_PROCS )
            {
                std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
                          << proc_config().proc_rank() << std::endl;
                std::cerr.flush();
                MPI_Abort( proc_config().proc_comm(), 66 );
            }
            sharing_procs.resize( MAX_SHARING_PROCS, -1 );
            sharing_handles.resize( MAX_SHARING_PROCS, 0 );
            result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
            result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
            result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
            sharedEnts.insert( this_ent );
        }
 
        // Reset sharing proc(s) tags
        sharing_procs.clear();
        sharing_handles.clear();
    }
 
#ifndef NDEBUG
    // Shouldn't be any repeated entities in any of the vectors in proc_nvecs
    for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
         mit != proc_nvecs.end(); ++mit )
    {
        std::vector< EntityHandle > tmp_vec = ( mit->second );
        std::sort( tmp_vec.begin(), tmp_vec.end() );
        std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
        assert( vit == tmp_vec.end() );
    }
#endif
 
    return MB_SUCCESS;
}
 
//! Get processors with which this processor communicates; sets are sorted by processor
ErrorCode ParallelComm::get_interface_procs( std::set< unsigned int >& procs_set, bool get_buffs )
{
    // Make sure the sharing procs vector is empty
    procs_set.clear();
 
    // Pre-load vector of single-proc tag values
    unsigned int i, j;
    std::vector< int > iface_proc( interfaceSets.size() );
    ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), interfaceSets, &iface_proc[0] );MB_CHK_SET_ERR( result, "Failed to get iface_proc for iface sets" );
 
    // Get sharing procs either from single-proc vector or by getting
    // multi-proc tag value
    int tmp_iface_procs[MAX_SHARING_PROCS];
    std::fill( tmp_iface_procs, tmp_iface_procs + MAX_SHARING_PROCS, -1 );
    Range::iterator rit;
    for( rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); ++rit, i++ )
    {
        if( -1 != iface_proc[i] )
        {
            assert( iface_proc[i] != (int)procConfig.proc_rank() );
            procs_set.insert( (unsigned int)iface_proc[i] );
        }
        else
        {
            // Get the sharing_procs tag
            result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, tmp_iface_procs );MB_CHK_SET_ERR( result, "Failed to get iface_procs for iface set" );
            for( j = 0; j < MAX_SHARING_PROCS; j++ )
            {
                if( -1 != tmp_iface_procs[j] && tmp_iface_procs[j] != (int)procConfig.proc_rank() )
                    procs_set.insert( (unsigned int)tmp_iface_procs[j] );
                else if( -1 == tmp_iface_procs[j] )
                {
                    std::fill( tmp_iface_procs, tmp_iface_procs + j, -1 );
                    break;
                }
            }
        }
    }
 
    if( get_buffs )
    {
        for( std::set< unsigned int >::iterator sit = procs_set.begin(); sit != procs_set.end(); ++sit )
            get_buffers( *sit );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_pstatus( EntityHandle entity, unsigned char& pstatus_val )
{
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstatus_val );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
    return result;
}
 
ErrorCode ParallelComm::get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents )
{
    Range ents;
    ErrorCode result;
 
    if( -1 == dim )
    {
        result = mbImpl->get_entities_by_handle( 0, ents );MB_CHK_SET_ERR( result, "Failed to get all entities" );
    }
    else
    {
        result = mbImpl->get_entities_by_dimension( 0, dim, ents );MB_CHK_SET_ERR( result, "Failed to get entities of dimension " << dim );
    }
 
    std::vector< unsigned char > pstatus( ents.size() );
    result = mbImpl->tag_get_data( pstatus_tag(), ents, &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
    Range::iterator rit = ents.begin();
    int i               = 0;
    if( pstatus_val )
    {
        for( ; rit != ents.end(); i++, ++rit )
        {
            if( pstatus[i] & pstatus_val && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
                pstatus_ents.insert( *rit );
        }
    }
    else
    {
        for( ; rit != ents.end(); i++, ++rit )
        {
            if( !pstatus[i] && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
                pstatus_ents.insert( *rit );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::check_global_ids( EntityHandle this_set,
                                          const int dimension,
                                          const int start_id,
                                          const bool largest_dim_only,
                                          const bool parallel,
                                          const bool owned_only )
{
    // Global id tag
    Tag gid_tag = mbImpl->globalId_tag();
    int def_val = -1;
    Range dum_range;
 
    void* tag_ptr    = &def_val;
    ErrorCode result = mbImpl->get_entities_by_type_and_tag( this_set, MBVERTEX, &gid_tag, &tag_ptr, 1, dum_range );MB_CHK_SET_ERR( result, "Failed to get entities by MBVERTEX type and gid tag" );
 
    if( !dum_range.empty() )
    {
        // Just created it, so we need global ids
        result = assign_global_ids( this_set, dimension, start_id, largest_dim_only, parallel, owned_only );MB_CHK_SET_ERR( result, "Failed assigning global ids" );
    }
 
    return MB_SUCCESS;
}
 
bool ParallelComm::is_iface_proc( EntityHandle this_set, int to_proc )
{
    int sharing_procs[MAX_SHARING_PROCS];
    std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
    ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &this_set, 1, sharing_procs );
    if( MB_SUCCESS == result && to_proc == sharing_procs[0] ) return true;
 
    result = mbImpl->tag_get_data( sharedps_tag(), &this_set, 1, sharing_procs );
    if( MB_SUCCESS != result ) return false;
 
    for( int i = 0; i < MAX_SHARING_PROCS; i++ )
    {
        if( to_proc == sharing_procs[i] )
            return true;
        else if( -1 == sharing_procs[i] )
            return false;
    }
 
    return false;
}
 
ErrorCode ParallelComm::filter_pstatus( Range& ents,
                                        unsigned char pstat,
                                        unsigned char op,
                                        int to_proc,
                                        Range* returned_ents )
{
    Range tmp_ents;
 
    // assert(!ents.empty());
    if( ents.empty() )
    {
        if( returned_ents ) returned_ents->clear();
        return MB_SUCCESS;
    }
 
    // Put into tmp_ents any entities which are not owned locally or
    // who are already shared with to_proc
    std::vector< unsigned char > shared_flags( ents.size() ), shared_flags2;
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), ents, &shared_flags[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus flag" );
    Range::const_iterator rit, hint = tmp_ents.begin();
    ;
    int i;
    if( op == PSTATUS_OR )
    {
        for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
        {
            if( ( ( shared_flags[i] & ~pstat ) ^ shared_flags[i] ) & pstat )
            {
                hint = tmp_ents.insert( hint, *rit );
                if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
            }
        }
    }
    else if( op == PSTATUS_AND )
    {
        for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
        {
            if( ( shared_flags[i] & pstat ) == pstat )
            {
                hint = tmp_ents.insert( hint, *rit );
                if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
            }
        }
    }
    else if( op == PSTATUS_NOT )
    {
        for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
        {
            if( !( shared_flags[i] & pstat ) )
            {
                hint = tmp_ents.insert( hint, *rit );
                if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
            }
        }
    }
    else
    {
        assert( false );
        return MB_FAILURE;
    }
 
    if( -1 != to_proc )
    {
        int sharing_procs[MAX_SHARING_PROCS];
        std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
        Range tmp_ents2;
        hint = tmp_ents2.begin();
 
        for( rit = tmp_ents.begin(), i = 0; rit != tmp_ents.end(); ++rit, i++ )
        {
            // We need to check sharing procs
            if( shared_flags2[i] & PSTATUS_MULTISHARED )
            {
                result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag" );
                assert( -1 != sharing_procs[0] );
                for( unsigned int j = 0; j < MAX_SHARING_PROCS; j++ )
                {
                    // If to_proc shares this entity, add it to list
                    if( sharing_procs[j] == to_proc )
                    {
                        hint = tmp_ents2.insert( hint, *rit );
                    }
                    else if( -1 == sharing_procs[j] )
                        break;
 
                    sharing_procs[j] = -1;
                }
            }
            else if( shared_flags2[i] & PSTATUS_SHARED )
            {
                result = mbImpl->tag_get_data( sharedp_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag" );
                assert( -1 != sharing_procs[0] );
                if( sharing_procs[0] == to_proc ) hint = tmp_ents2.insert( hint, *rit );
                sharing_procs[0] = -1;
            }
            else
                assert( "should never get here" && false );
        }
 
        tmp_ents.swap( tmp_ents2 );
    }
 
    if( returned_ents )
        returned_ents->swap( tmp_ents );
    else
        ents.swap( tmp_ents );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::exchange_ghost_cells( int ghost_dim,
                                              int bridge_dim,
                                              int num_layers,
                                              int addl_ents,
                                              bool store_remote_handles,
                                              bool wait_all,
                                              EntityHandle* file_set )
{
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        if( !num_layers )
            MPE_Log_event( IFACE_START, procConfig.proc_rank(), "Starting interface exchange." );
        else
            MPE_Log_event( GHOST_START, procConfig.proc_rank(), "Starting ghost exchange." );
    }
#endif
 
    myDebug->tprintf( 1, "Entering exchange_ghost_cells with num_layers = %d\n", num_layers );
    if( myDebug->get_verbosity() == 4 )
    {
        msgs.clear();
        msgs.reserve( MAX_SHARING_PROCS );
    }
 
    // If we're only finding out about existing ents, we have to be storing
    // remote handles too
    assert( num_layers > 0 || store_remote_handles );
 
    const bool is_iface = !num_layers;
 
    // Get the b-dimensional interface(s) with with_proc, where b = bridge_dim
 
    int success;
    ErrorCode result = MB_SUCCESS;
    int incoming1 = 0, incoming2 = 0;
 
    reset_all_buffers();
 
    // When this function is called, buffProcs should already have any
    // communicating procs
 
    //===========================================
    // Post ghost irecv's for ghost entities from all communicating procs
    //===========================================
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." );
    }
#endif
 
    // Index reqs the same as buffer/sharing procs indices
    std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ),
        recv_remoteh_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    std::vector< unsigned int >::iterator proc_it;
    int ind, p;
    sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
    {
        incoming1++;
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_ENTS_SIZE, incoming1 );
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
                             MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
        }
    }
 
    //===========================================
    // Get entities to be sent to neighbors
    //===========================================
    Range sent_ents[MAX_SHARING_PROCS], allsent, tmp_range;
    TupleList entprocs;
    int dum_ack_buff;
    result = get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents, allsent, entprocs );MB_CHK_SET_ERR( result, "get_sent_ents failed" );
 
    // augment file set with the entities to be sent
    // we might have created new entities if addl_ents>0, edges and/or faces
    if( addl_ents > 0 && file_set && !allsent.empty() )
    {
        result = mbImpl->add_entities( *file_set, allsent );MB_CHK_SET_ERR( result, "Failed to add new sub-entities to set" );
    }
    myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
                      (unsigned long)allsent.size() );
 
    //===========================================
    // Pack and send ents from this proc to others
    //===========================================
    for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
    {
        myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", sent_ents[p].compactness(),
                          (unsigned long)sent_ents[p].size() );
 
        // Reserve space on front for size and for initial buff size
        localOwnedBuffs[p]->reset_buffer( sizeof( int ) );
 
        // Entities
        result = pack_entities( sent_ents[p], localOwnedBuffs[p], store_remote_handles, buffProcs[p], is_iface,
                                &entprocs, &allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *localOwnedBuffs[p] );
        }
 
        // Send the buffer (size stored in front in send_buffer)
        result = send_buffer( *proc_it, localOwnedBuffs[p], MB_MESG_ENTS_SIZE, sendReqs[3 * p],
                              recv_ent_reqs[3 * p + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE,
                              ( !is_iface && store_remote_handles ?  // this used for ghosting only
                                    localOwnedBuffs[p]
                                                                  : NULL ),
                              &recv_remoteh_reqs[3 * p], &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" );
    }
 
    entprocs.reset();
 
    //===========================================
    // Receive/unpack new entities
    //===========================================
    // Number of incoming messages for ghosts is the number of procs we
    // communicate with; for iface, it's the number of those with lower rank
    MPI_Status status;
    std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
    std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
    std::vector< std::vector< int > > L1p( buffProcs.size() );
    std::vector< EntityHandle > L2hloc, L2hrem;
    std::vector< unsigned int > L2p;
    std::vector< EntityHandle > new_ents;
 
    while( incoming1 )
    {
        // Wait for all recvs of ghost ents before proceeding to sending remote handles,
        // b/c some procs may have sent to a 3rd proc ents owned by me;
        PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
 
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
        }
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming1--;
        bool done = false;
 
        // In case ind is for ack, we need index of one before it
        unsigned int base_ind = 3 * ( ind / 3 );
        result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1],
                              recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1],
                              sendReqs[base_ind + 2], done,
                              ( !is_iface && store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ),
                              MB_MESG_REMOTEH_SIZE,  // maybe base_ind+1?
                              &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
 
        if( done )
        {
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] );
            }
 
            // Message completely received - process buffer that was sent
            remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
            result = unpack_entities( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, ind / 3, is_iface,
                                      L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );
            if( MB_SUCCESS != result )
            {
                std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
                print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false );
                return result;
            }
 
            if( recv_ent_reqs.size() != 3 * buffProcs.size() )
            {
                // Post irecv's for remote handles from new proc; shouldn't be iface,
                // since we know about all procs we share with
                assert( !is_iface );
                recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
                for( unsigned int i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 )
                {
                    localOwnedBuffs[i / 3]->reset_buffer();
                    incoming2++;
                    PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr,
                                       INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
                    success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
                                         buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
                                         &recv_remoteh_reqs[i] );
                    if( success != MPI_SUCCESS )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
                    }
                }
                recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
                sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
            }
        }
    }
 
    // Add requests for any new addl procs
    if( recv_ent_reqs.size() != 3 * buffProcs.size() )
    {
        // Shouldn't get here...
        MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in ghost exchange" );
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." );
    }
#endif
 
    if( is_iface )
    {
        // Need to check over entities I sent and make sure I received
        // handles for them from all expected procs; if not, need to clean
        // them up
        result = check_clean_iface( allsent );
        if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
 
        // Now set the shared/interface tag on non-vertex entities on interface
        result = tag_iface_entities();MB_CHK_SET_ERR( result, "Failed to tag iface entities" );
 
#ifndef NDEBUG
        result = check_sent_ents( allsent );
        if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
        result = check_all_shared_handles( true );
        if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
#endif
 
#ifdef MOAB_HAVE_MPE
        if( myDebug->get_verbosity() == 2 )
        {
            MPE_Log_event( IFACE_END, procConfig.proc_rank(), "Ending interface exchange." );
        }
#endif
 
        //===========================================
        // Wait if requested
        //===========================================
        if( wait_all )
        {
            if( myDebug->get_verbosity() == 5 )
            {
                success = MPI_Barrier( procConfig.proc_comm() );
            }
            else
            {
                MPI_Status mult_status[3 * MAX_SHARING_PROCS];
                success = MPI_Waitall( 3 * buffProcs.size(), &recv_ent_reqs[0], mult_status );
                if( MPI_SUCCESS != success )
                {
                    MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
                }
                success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
                if( MPI_SUCCESS != success )
                {
                    MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
                }
                /*success = MPI_Waitall(3*buffProcs.size(), &recv_remoteh_reqs[0], mult_status);
                if (MPI_SUCCESS != success) {
                  MB_SET_ERR(MB_FAILURE, "Failed in waitall in ghost exchange");
                }*/
            }
        }
 
        myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
        myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==true \n" );
 
        return MB_SUCCESS;
    }
 
    // we still need to wait on sendReqs, if they are not fulfilled yet
    if( wait_all )
    {
        if( myDebug->get_verbosity() == 5 )
        {
            success = MPI_Barrier( procConfig.proc_comm() );
        }
        else
        {
            MPI_Status mult_status[3 * MAX_SHARING_PROCS];
            success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
            if( MPI_SUCCESS != success )
            {
                MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
            }
        }
    }
    //===========================================
    // Send local handles for new ghosts to owner, then add
    // those to ghost list for that owner
    //===========================================
    for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
    {
 
        // Reserve space on front for size and for initial buff size
        remoteOwnedBuffs[p]->reset_buffer( sizeof( int ) );
 
        result = pack_remote_handles( L1hloc[p], L1hrem[p], L1p[p], *proc_it, remoteOwnedBuffs[p] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
        remoteOwnedBuffs[p]->set_stored_size();
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *remoteOwnedBuffs[p] );
        }
        result = send_buffer( buffProcs[p], remoteOwnedBuffs[p], MB_MESG_REMOTEH_SIZE, sendReqs[3 * p],
                              recv_remoteh_reqs[3 * p + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
    }
 
    //===========================================
    // Process remote handles of my ghosteds
    //===========================================
    while( incoming2 )
    {
        PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
        }
 
        // OK, received something; decrement incoming counter
        incoming2--;
 
        PRINT_DEBUG_RECD( status );
 
        bool done             = false;
        unsigned int base_ind = 3 * ( ind / 3 );
        result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1],
                              recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3],
                              sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
        if( done )
        {
            // Incoming remote handles
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] );
            }
            localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
            result =
                unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
        }
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
        MPE_Log_event( GHOST_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." );
    }
#endif
 
    //===========================================
    // Wait if requested
    //===========================================
    if( wait_all )
    {
        if( myDebug->get_verbosity() == 5 )
        {
            success = MPI_Barrier( procConfig.proc_comm() );
        }
        else
        {
            MPI_Status mult_status[3 * MAX_SHARING_PROCS];
            success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status );
            if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
        }
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
        }
    }
 
#ifndef NDEBUG
    result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" );
    result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Failed check on all shared handles" );
#endif
 
    if( file_set && !new_ents.empty() )
    {
        result = mbImpl->add_entities( *file_set, &new_ents[0], new_ents.size() );MB_CHK_SET_ERR( result, "Failed to add new entities to set" );
    }
 
    myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
    myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==false \n" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::send_buffer( const unsigned int to_proc,
                                     Buffer* send_buff,
                                     int mesg_tag,
                                     MPI_Request& send_req,
                                     MPI_Request& ack_req,
                                     int* ack_buff,
                                     int& this_incoming,
                                     int next_mesg_tag,
                                     Buffer* next_recv_buff,
                                     MPI_Request* next_recv_req,
                                     int* next_incoming )
{
    ErrorCode result = MB_SUCCESS;
    int success;
 
    // If small message, post recv for remote handle message
    if( send_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE && next_recv_buff )
    {
        ( *next_incoming )++;
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, next_mesg_tag,
                           *next_incoming );
        success = MPI_Irecv( next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, to_proc, next_mesg_tag,
                             procConfig.proc_comm(), next_recv_req );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv for next message in ghost exchange" );
        }
    }
    // If large, we'll need an ack before sending the rest
    else if( send_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
    {
        this_incoming++;
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, (unsigned char*)ack_buff, sizeof( int ), mesg_tag - 1,
                           this_incoming );
        success = MPI_Irecv( (void*)ack_buff, sizeof( int ), MPI_UNSIGNED_CHAR, to_proc, mesg_tag - 1,
                             procConfig.proc_comm(), &ack_req );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv for entity ack in ghost exchange" );
        }
    }
 
    // Send the buffer
    PRINT_DEBUG_ISEND( procConfig.proc_rank(), to_proc, send_buff->mem_ptr, mesg_tag,
                       std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ) );
    assert( 0 <= send_buff->get_stored_size() && send_buff->get_stored_size() <= (int)send_buff->alloc_size );
    success = MPI_Isend( send_buff->mem_ptr, std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ),
                         MPI_UNSIGNED_CHAR, to_proc, mesg_tag, procConfig.proc_comm(), &send_req );
    if( success != MPI_SUCCESS ) return MB_FAILURE;
 
    return result;
}
 
ErrorCode ParallelComm::recv_buffer( int mesg_tag_expected,
                                     const MPI_Status& mpi_status,
                                     Buffer* recv_buff,
                                     MPI_Request& recv_req,
                                     MPI_Request& /*ack_recvd_req*/,
                                     int& this_incoming,
                                     Buffer* send_buff,
                                     MPI_Request& send_req,
                                     MPI_Request& sent_ack_req,
                                     bool& done,
                                     Buffer* next_buff,
                                     int next_tag,
                                     MPI_Request* next_req,
                                     int* next_incoming )
{
    // Process a received message; if there will be more coming,
    // post a receive for 2nd part then send an ack message
    int from_proc = mpi_status.MPI_SOURCE;
    int success;
 
    // Set the buff_ptr on the recv_buffer; needs to point beyond any
    // valid data already in the buffer
    recv_buff->reset_ptr( std::min( recv_buff->get_stored_size(), (int)recv_buff->alloc_size ) );
 
    if( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
    {
        // 1st message & large - allocate buffer, post irecv for 2nd message,
        // then send ack
        recv_buff->reserve( recv_buff->get_stored_size() );
        assert( recv_buff->alloc_size > INITIAL_BUFF_SIZE );
 
        // Will expect a 2nd message
        this_incoming++;
 
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr + INITIAL_BUFF_SIZE,
                           recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, mesg_tag_expected + 1, this_incoming );
        success = MPI_Irecv( recv_buff->mem_ptr + INITIAL_BUFF_SIZE, recv_buff->get_stored_size() - INITIAL_BUFF_SIZE,
                             MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &recv_req );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post 2nd iRecv in ghost exchange" );
        }
 
        // Send ack, doesn't matter what data actually is
        PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr, mesg_tag_expected - 1,
                           sizeof( int ) );
        success = MPI_Isend( recv_buff->mem_ptr, sizeof( int ), MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected - 1,
                             procConfig.proc_comm(), &sent_ack_req );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to send ack in ghost exchange" );
        }
    }
    else if( mpi_status.MPI_TAG == mesg_tag_expected - 1 )
    {
        // Got an ack back, send the 2nd half of message
 
        // Should be a large message if we got this
        assert( *( (size_t*)send_buff->mem_ptr ) > INITIAL_BUFF_SIZE );
 
        // Post irecv for next message, then send 2nd message
        if( next_buff )
        {
            // We'll expect a return message
            ( *next_incoming )++;
            PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, next_buff->mem_ptr, INITIAL_BUFF_SIZE, next_tag,
                               *next_incoming );
 
            success = MPI_Irecv( next_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, next_tag,
                                 procConfig.proc_comm(), next_req );
            if( success != MPI_SUCCESS )
            {
                MB_SET_ERR( MB_FAILURE, "Failed to post next irecv in ghost exchange" );
            }
        }
 
        // Send 2nd message
        PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, send_buff->mem_ptr + INITIAL_BUFF_SIZE,
                           mesg_tag_expected + 1, send_buff->get_stored_size() - INITIAL_BUFF_SIZE );
 
        assert( send_buff->get_stored_size() - INITIAL_BUFF_SIZE < send_buff->alloc_size &&
                0 <= send_buff->get_stored_size() );
        success = MPI_Isend( send_buff->mem_ptr + INITIAL_BUFF_SIZE, send_buff->get_stored_size() - INITIAL_BUFF_SIZE,
                             MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &send_req );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to send 2nd message in ghost exchange" );
        }
    }
    else if( ( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE ) ||
             mpi_status.MPI_TAG == mesg_tag_expected + 1 )
    {
        // Message completely received - signal that we're done
        done = true;
    }
 
    return MB_SUCCESS;
}
 
struct ProcList
{
    int procs[MAX_SHARING_PROCS];
};
static bool operator<( const ProcList& a, const ProcList& b )
{
    for( int i = 0; i < MAX_SHARING_PROCS; i++ )
    {
        if( a.procs[i] < b.procs[i] )
            return true;
        else if( b.procs[i] < a.procs[i] )
            return false;
        else if( a.procs[i] < 0 )
            return false;
    }
    return false;
}
 
ErrorCode ParallelComm::check_clean_iface( Range& allsent )
{
    // allsent is all entities I think are on interface; go over them, looking
    // for zero-valued handles, and fix any I find
 
    // Keep lists of entities for which teh sharing data changed, grouped
    // by set of sharing procs.
    typedef std::map< ProcList, Range > procmap_t;
    procmap_t old_procs, new_procs;
 
    ErrorCode result = MB_SUCCESS;
    Range::iterator rit;
    Range::reverse_iterator rvit;
    unsigned char pstatus;
    int nump;
    ProcList sharedp;
    EntityHandle sharedh[MAX_SHARING_PROCS];
    for( rvit = allsent.rbegin(); rvit != allsent.rend(); ++rvit )
    {
        result = get_sharing_data( *rvit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
        assert( "Should be shared with at least one other proc" &&
                ( nump > 1 || sharedp.procs[0] != (int)procConfig.proc_rank() ) );
        assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
 
        // Look for first null handle in list
        int idx = std::find( sharedh, sharedh + nump, (EntityHandle)0 ) - sharedh;
        if( idx == nump ) continue;  // All handles are valid
 
        ProcList old_list( sharedp );
        std::sort( old_list.procs, old_list.procs + nump );
        old_procs[old_list].insert( *rvit );
 
        // Remove null handles and corresponding proc ranks from lists
        int new_nump       = idx;
        bool removed_owner = !idx;
        for( ++idx; idx < nump; ++idx )
        {
            if( sharedh[idx] )
            {
                sharedh[new_nump]       = sharedh[idx];
                sharedp.procs[new_nump] = sharedp.procs[idx];
                ++new_nump;
            }
        }
        sharedp.procs[new_nump] = -1;
 
        if( removed_owner && new_nump > 1 )
        {
            // The proc that we choose as the entity owner isn't sharing the
            // entity (doesn't have a copy of it). We need to pick a different
            // owner. Choose the proc with lowest rank.
            idx = std::min_element( sharedp.procs, sharedp.procs + new_nump ) - sharedp.procs;
            std::swap( sharedp.procs[0], sharedp.procs[idx] );
            std::swap( sharedh[0], sharedh[idx] );
            if( sharedp.procs[0] == (int)proc_config().proc_rank() ) pstatus &= ~PSTATUS_NOT_OWNED;
        }
 
        result = set_sharing_data( *rvit, pstatus, nump, new_nump, sharedp.procs, sharedh );MB_CHK_SET_ERR( result, "Failed to set sharing data in check_clean_iface" );
 
        if( new_nump > 1 )
        {
            if( new_nump == 2 )
            {
                if( sharedp.procs[1] != (int)proc_config().proc_rank() )
                {
                    assert( sharedp.procs[0] == (int)proc_config().proc_rank() );
                    sharedp.procs[0] = sharedp.procs[1];
                }
                sharedp.procs[1] = -1;
            }
            else
            {
                std::sort( sharedp.procs, sharedp.procs + new_nump );
            }
            new_procs[sharedp].insert( *rvit );
        }
    }
 
    if( old_procs.empty() )
    {
        assert( new_procs.empty() );
        return MB_SUCCESS;
    }
 
    // Update interface sets
    procmap_t::iterator pmit;
    // std::vector<unsigned char> pstatus_list;
    rit = interface_sets().begin();
    while( rit != interface_sets().end() )
    {
        result = get_sharing_data( *rit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data for interface set" );
        assert( nump != 2 );
        std::sort( sharedp.procs, sharedp.procs + nump );
        assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
 
        pmit = old_procs.find( sharedp );
        if( pmit != old_procs.end() )
        {
            result = mbImpl->remove_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
        }
 
        pmit = new_procs.find( sharedp );
        if( pmit == new_procs.end() )
        {
            int count;
            result = mbImpl->get_number_entities_by_handle( *rit, count );MB_CHK_SET_ERR( result, "Failed to get number of entities in interface set" );
            if( !count )
            {
                result = mbImpl->delete_entities( &*rit, 1 );MB_CHK_SET_ERR( result, "Failed to delete entities from interface set" );
                rit = interface_sets().erase( rit );
            }
            else
            {
                ++rit;
            }
        }
        else
        {
            result = mbImpl->add_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
 
            // Remove those that we've processed so that we know which ones
            // are new.
            new_procs.erase( pmit );
            ++rit;
        }
    }
 
    // Create interface sets for new proc id combinations
    std::fill( sharedh, sharedh + MAX_SHARING_PROCS, 0 );
    for( pmit = new_procs.begin(); pmit != new_procs.end(); ++pmit )
    {
        EntityHandle new_set;
        result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
        interfaceSets.insert( new_set );
 
        // Add entities
        result = mbImpl->add_entities( new_set, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
        // Tag set with the proc rank(s)
        assert( pmit->first.procs[0] >= 0 );
        pstatus = PSTATUS_SHARED | PSTATUS_INTERFACE;
        if( pmit->first.procs[1] == -1 )
        {
            int other = pmit->first.procs[0];
            assert( other != (int)procConfig.proc_rank() );
            result = mbImpl->tag_set_data( sharedp_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            sharedh[0] = 0;
            result     = mbImpl->tag_set_data( sharedh_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            if( other < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
        }
        else
        {
            result = mbImpl->tag_set_data( sharedps_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            result = mbImpl->tag_set_data( sharedhs_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
            pstatus |= PSTATUS_MULTISHARED;
            if( pmit->first.procs[0] < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
        }
 
        result = mbImpl->tag_set_data( pstatus_tag(), &new_set, 1, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
 
        // Set pstatus on all interface entities in set
        result = mbImpl->tag_clear_data( pstatus_tag(), pmit->second, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface entities with pstatus" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::set_sharing_data( EntityHandle ent,
                                          unsigned char pstatus,
                                          int old_nump,
                                          int new_nump,
                                          int* ps,
                                          EntityHandle* hs )
{
    // If new nump is less than 3, the entity is no longer mutishared
    if( old_nump > 2 && ( pstatus & PSTATUS_MULTISHARED ) && new_nump < 3 )
    {
        // Unset multishared flag
        pstatus ^= PSTATUS_MULTISHARED;
    }
 
    // Check for consistency in input data
    // DBG
    /*  bool con1 = ((new_nump == 2 && pstatus&PSTATUS_SHARED && !(pstatus&PSTATUS_MULTISHARED)) ||
      (new_nump > 2 && pstatus&PSTATUS_SHARED && pstatus&PSTATUS_MULTISHARED)); bool con2 =
      (!(pstatus&PSTATUS_GHOST) || pstatus&PSTATUS_SHARED); bool con3 = (new_nump < 3 ||
      (pstatus&PSTATUS_NOT_OWNED && ps[0] != (int)rank()) || (!(pstatus&PSTATUS_NOT_OWNED) && ps[0]
      == (int)rank())); std::cout<<"current rank = "<<rank()<<std::endl; std::cout<<"condition
      1::"<<con1<<std::endl; std::cout<<"condition 2::"<<con2<<std::endl; std::cout<<"condition
      3::"<<con3<<std::endl;*/
 
    // DBG
 
    assert( new_nump > 1 &&
            ( ( new_nump == 2 && pstatus & PSTATUS_SHARED &&
                !( pstatus & PSTATUS_MULTISHARED ) ) ||  // If <= 2 must not be multishared
              ( new_nump > 2 && pstatus & PSTATUS_SHARED &&
                pstatus & PSTATUS_MULTISHARED ) ) &&                         // If > 2 procs, must be multishared
            ( !( pstatus & PSTATUS_GHOST ) || pstatus & PSTATUS_SHARED ) &&  // If ghost, it must also be shared
            ( new_nump < 3 ||
              ( pstatus & PSTATUS_NOT_OWNED && ps[0] != (int)rank() ) ||      // I'm not owner and first proc not me
              ( !( pstatus & PSTATUS_NOT_OWNED ) && ps[0] == (int)rank() ) )  // I'm owner and first proc is me
    );
 
#ifndef NDEBUG
    {
        // Check for duplicates in proc list
        std::set< unsigned int > dumprocs;
        int dp = 0;
        for( ; dp < old_nump && -1 != ps[dp]; dp++ )
            dumprocs.insert( ps[dp] );
        assert( dp == (int)dumprocs.size() );
    }
#endif
 
    ErrorCode result;
    // Reset any old data that needs to be
    if( old_nump > 2 && new_nump < 3 )
    {
        // Need to remove multishared tags
        result = mbImpl->tag_delete_data( sharedps_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:1" );
        result = mbImpl->tag_delete_data( sharedhs_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:2" );
        //    if (new_nump < 2)
        //      pstatus = 0x0;
        //    else if (ps[0] != (int)proc_config().proc_rank())
        //      pstatus |= PSTATUS_NOT_OWNED;
    }
    else if( ( old_nump < 3 && new_nump > 2 ) || ( old_nump > 1 && new_nump == 1 ) )
    {
        // Reset sharedp and sharedh tags
        int tmp_p          = -1;
        EntityHandle tmp_h = 0;
        result             = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, &tmp_p );MB_CHK_SET_ERR( result, "set_sharing_data:3" );
        result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, &tmp_h );MB_CHK_SET_ERR( result, "set_sharing_data:4" );
    }
 
    assert( "check for multishared/owner I'm first proc" &&
            ( !( pstatus & PSTATUS_MULTISHARED ) || ( pstatus & ( PSTATUS_NOT_OWNED | PSTATUS_GHOST ) ) ||
              ( ps[0] == (int)rank() ) ) &&
            "interface entities should have > 1 proc" && ( !( pstatus & PSTATUS_INTERFACE ) || new_nump > 1 ) &&
            "ghost entities should have > 1 proc" && ( !( pstatus & PSTATUS_GHOST ) || new_nump > 1 ) );
 
    // Now set new data
    if( new_nump > 2 )
    {
        result = mbImpl->tag_set_data( sharedps_tag(), &ent, 1, ps );MB_CHK_SET_ERR( result, "set_sharing_data:5" );
        result = mbImpl->tag_set_data( sharedhs_tag(), &ent, 1, hs );MB_CHK_SET_ERR( result, "set_sharing_data:6" );
    }
    else
    {
        unsigned int j = ( ps[0] == (int)procConfig.proc_rank() ? 1 : 0 );
        assert( -1 != ps[j] );
        result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, ps + j );MB_CHK_SET_ERR( result, "set_sharing_data:7" );
        result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, hs + j );MB_CHK_SET_ERR( result, "set_sharing_data:8" );
    }
 
    result = mbImpl->tag_set_data( pstatus_tag(), &ent, 1, &pstatus );MB_CHK_SET_ERR( result, "set_sharing_data:9" );
 
    if( old_nump > 1 && new_nump < 2 ) sharedEnts.erase( ent );
 
    return result;
}
 
ErrorCode ParallelComm::get_sent_ents( const bool is_iface,
                                       const int bridge_dim,
                                       const int ghost_dim,
                                       const int num_layers,
                                       const int addl_ents,
                                       Range* sent_ents,
                                       Range& allsent,
                                       TupleList& entprocs )
{
    ErrorCode result;
    unsigned int ind;
    std::vector< unsigned int >::iterator proc_it;
    Range tmp_range;
 
    // Done in a separate loop over procs because sometimes later procs
    // need to add info to earlier procs' messages
    for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
    {
        if( !is_iface )
        {
            result =
                get_ghosted_entities( bridge_dim, ghost_dim, buffProcs[ind], num_layers, addl_ents, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get ghost layers" );
        }
        else
        {
            result = get_iface_entities( buffProcs[ind], -1, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get interface layers" );
        }
 
        // Filter out entities already shared with destination
        tmp_range.clear();
        result = filter_pstatus( sent_ents[ind], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
        if( !tmp_range.empty() ) sent_ents[ind] = subtract( sent_ents[ind], tmp_range );
 
        allsent.merge( sent_ents[ind] );
    }
 
    //===========================================
    // Need to get procs each entity is sent to
    //===========================================
 
    // Get the total # of proc/handle pairs
    int npairs = 0;
    for( ind = 0; ind < buffProcs.size(); ind++ )
        npairs += sent_ents[ind].size();
 
    // Allocate a TupleList of that size
    entprocs.initialize( 1, 0, 1, 0, npairs );
    entprocs.enableWriteAccess();
 
    // Put the proc/handle pairs in the list
    for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
    {
        for( Range::iterator rit = sent_ents[ind].begin(); rit != sent_ents[ind].end(); ++rit )
        {
            entprocs.vi_wr[entprocs.get_n()]  = *proc_it;
            entprocs.vul_wr[entprocs.get_n()] = *rit;
            entprocs.inc_n();
        }
    }
    // Sort by handle
    moab::TupleList::buffer sort_buffer;
    sort_buffer.buffer_init( npairs );
    entprocs.sort( 1, &sort_buffer );
 
    entprocs.disableWriteAccess();
    sort_buffer.reset();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::exchange_ghost_cells( ParallelComm** pcs,
                                              unsigned int num_procs,
                                              int ghost_dim,
                                              int bridge_dim,
                                              int num_layers,
                                              int addl_ents,
                                              bool store_remote_handles,
                                              EntityHandle* file_sets )
{
    // Static version of function, exchanging info through buffers rather
    // than through messages
 
    // If we're only finding out about existing ents, we have to be storing
    // remote handles too
    assert( num_layers > 0 || store_remote_handles );
 
    const bool is_iface = !num_layers;
 
    unsigned int ind;
    ParallelComm* pc;
    ErrorCode result = MB_SUCCESS;
 
    std::vector< Error* > ehs( num_procs );
    for( unsigned int i = 0; i < num_procs; i++ )
    {
        result = pcs[i]->get_moab()->query_interface( ehs[i] );
        assert( MB_SUCCESS == result );
    }
 
    // When this function is called, buffProcs should already have any
    // communicating procs
 
    //===========================================
    // Get entities to be sent to neighbors
    //===========================================
 
    // Done in a separate loop over procs because sometimes later procs
    // need to add info to earlier procs' messages
    Range sent_ents[MAX_SHARING_PROCS][MAX_SHARING_PROCS], allsent[MAX_SHARING_PROCS];
 
    //===========================================
    // Get entities to be sent to neighbors
    //===========================================
    TupleList entprocs[MAX_SHARING_PROCS];
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        pc     = pcs[p];
        result = pc->get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents[p], allsent[p],
                                    entprocs[p] );MB_CHK_SET_ERR( result, "p = " << p << ", get_sent_ents failed" );
 
        //===========================================
        // Pack entities into buffers
        //===========================================
        for( ind = 0; ind < pc->buffProcs.size(); ind++ )
        {
            // Entities
            pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = pc->pack_entities( sent_ents[p][ind], pc->localOwnedBuffs[ind], store_remote_handles,
                                        pc->buffProcs[ind], is_iface, &entprocs[p], &allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", packing entities failed" );
        }
 
        entprocs[p].reset();
    }
 
    //===========================================
    // Receive/unpack new entities
    //===========================================
    // Number of incoming messages for ghosts is the number of procs we
    // communicate with; for iface, it's the number of those with lower rank
    std::vector< std::vector< EntityHandle > > L1hloc[MAX_SHARING_PROCS], L1hrem[MAX_SHARING_PROCS];
    std::vector< std::vector< int > > L1p[MAX_SHARING_PROCS];
    std::vector< EntityHandle > L2hloc[MAX_SHARING_PROCS], L2hrem[MAX_SHARING_PROCS];
    std::vector< unsigned int > L2p[MAX_SHARING_PROCS];
    std::vector< EntityHandle > new_ents[MAX_SHARING_PROCS];
 
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        L1hloc[p].resize( pcs[p]->buffProcs.size() );
        L1hrem[p].resize( pcs[p]->buffProcs.size() );
        L1p[p].resize( pcs[p]->buffProcs.size() );
    }
 
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        pc = pcs[p];
 
        for( ind = 0; ind < pc->buffProcs.size(); ind++ )
        {
            // Incoming ghost entities; unpack; returns entities received
            // both from sending proc and from owning proc (which may be different)
 
            // Buffer could be empty, which means there isn't any message to
            // unpack (due to this comm proc getting added as a result of indirect
            // communication); just skip this unpack
            if( pc->localOwnedBuffs[ind]->get_stored_size() == 0 ) continue;
 
            unsigned int to_p = pc->buffProcs[ind];
            pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = pcs[to_p]->unpack_entities( pc->localOwnedBuffs[ind]->buff_ptr, store_remote_handles, ind,
                                                 is_iface, L1hloc[to_p], L1hrem[to_p], L1p[to_p], L2hloc[to_p],
                                                 L2hrem[to_p], L2p[to_p], new_ents[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack entities" );
        }
    }
 
    if( is_iface )
    {
        // Need to check over entities I sent and make sure I received
        // handles for them from all expected procs; if not, need to clean
        // them up
        for( unsigned int p = 0; p < num_procs; p++ )
        {
            result = pcs[p]->check_clean_iface( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
        }
 
#ifndef NDEBUG
        for( unsigned int p = 0; p < num_procs; p++ )
        {
            result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
        }
        result = check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
#endif
        return MB_SUCCESS;
    }
 
    //===========================================
    // Send local handles for new ghosts to owner, then add
    // those to ghost list for that owner
    //===========================================
    std::vector< unsigned int >::iterator proc_it;
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        pc = pcs[p];
 
        for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
        {
            // Skip if iface layer and higher-rank proc
            pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = pc->pack_remote_handles( L1hloc[p][ind], L1hrem[p][ind], L1p[p][ind], *proc_it,
                                              pc->localOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to pack remote handles" );
        }
    }
 
    //===========================================
    // Process remote handles of my ghosteds
    //===========================================
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        pc = pcs[p];
 
        for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
        {
            // Incoming remote handles
            unsigned int to_p = pc->buffProcs[ind];
            pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = pcs[to_p]->unpack_remote_handles( p, pc->localOwnedBuffs[ind]->buff_ptr, L2hloc[to_p],
                                                       L2hrem[to_p], L2p[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack remote handles" );
        }
    }
 
#ifndef NDEBUG
    for( unsigned int p = 0; p < num_procs; p++ )
    {
        result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
    }
 
    result = ParallelComm::check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
#endif
 
    if( file_sets )
    {
        for( unsigned int p = 0; p < num_procs; p++ )
        {
            if( new_ents[p].empty() ) continue;
            result = pcs[p]->get_moab()->add_entities( file_sets[p], &new_ents[p][0], new_ents[p].size() );MB_CHK_SET_ERR( result, "p = " << p << ", failed to add new entities to set" );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& exchange_procs )
{
    // Set buffers
    int n_proc = exchange_procs.size();
    for( int i = 0; i < n_proc; i++ )
        get_buffers( exchange_procs[i] );
    reset_all_buffers();
 
    // Post ghost irecv's for entities from all communicating procs
    // Index requests the same as buffer/sharing procs indices
    int success;
    recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
    recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
    sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
 
    int incoming = 0;
    for( int i = 0; i < n_proc; i++ )
    {
        int ind = get_buffers( exchange_procs[i] );
        incoming++;
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_ENTS_SIZE, incoming );
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
                             MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs )
{
    // Set buffers
    int num = shared_procs.size();
    for( int i = 0; i < num; i++ )
        get_buffers( shared_procs[i] );
    reset_all_buffers();
    num = remoteOwnedBuffs.size();
    for( int i = 0; i < num; i++ )
        remoteOwnedBuffs[i]->set_stored_size();
    num = localOwnedBuffs.size();
    for( int i = 0; i < num; i++ )
        localOwnedBuffs[i]->set_stored_size();
 
    // Post ghost irecv's for entities from all communicating procs
    // Index requests the same as buffer/sharing procs indices
    int success;
    recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
    recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
    sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
 
    int incoming                           = 0;
    std::set< unsigned int >::iterator it  = recv_procs.begin();
    std::set< unsigned int >::iterator eit = recv_procs.end();
    for( ; it != eit; ++it )
    {
        int ind = get_buffers( *it );
        incoming++;
        PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_ENTS_SIZE, incoming );
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
                             MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::exchange_owned_meshs( std::vector< unsigned int >& exchange_procs,
                                              std::vector< Range* >& exchange_ents,
                                              std::vector< MPI_Request >& recv_ent_reqs,
                                              std::vector< MPI_Request >& recv_remoteh_reqs,
                                              bool store_remote_handles,
                                              bool wait_all,
                                              bool migrate,
                                              int dim )
{
    // Filter out entities already shared with destination
    // Exchange twice for entities and sets
    ErrorCode result;
    std::vector< unsigned int > exchange_procs_sets;
    std::vector< Range* > exchange_sets;
    int n_proc = exchange_procs.size();
    for( int i = 0; i < n_proc; i++ )
    {
        Range set_range   = exchange_ents[i]->subset_by_type( MBENTITYSET );
        *exchange_ents[i] = subtract( *exchange_ents[i], set_range );
        Range* tmp_range  = new Range( set_range );
        exchange_sets.push_back( tmp_range );
        exchange_procs_sets.push_back( exchange_procs[i] );
    }
 
    if( dim == 2 )
    {
        // Exchange entities first
        result = exchange_owned_mesh( exchange_procs, exchange_ents, recvReqs, recvRemotehReqs, true,
                                      store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" );
 
        // Exchange sets
        result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recvReqs, recvRemotehReqs, false,
                                      store_remote_handles, wait_all, migrate );
    }
    else
    {
        // Exchange entities first
        result = exchange_owned_mesh( exchange_procs, exchange_ents, recv_ent_reqs, recv_remoteh_reqs, false,
                                      store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" );
 
        // Exchange sets
        result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recv_ent_reqs, recv_remoteh_reqs, false,
                                      store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh sets" );
    }
 
    for( int i = 0; i < n_proc; i++ )
        delete exchange_sets[i];
 
    // Build up the list of shared entities
    std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
    int procs[MAX_SHARING_PROCS];
    EntityHandle handles[MAX_SHARING_PROCS];
    int nprocs;
    unsigned char pstat;
    for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
    {
        if( mbImpl->dimension_from_handle( *vit ) > 2 ) continue;
        result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data in exchange_owned_meshs" );
        std::sort( procs, procs + nprocs );
        std::vector< int > tmp_procs( procs, procs + nprocs );
        assert( tmp_procs.size() != 2 );
        proc_nvecs[tmp_procs].push_back( *vit );
    }
 
    // Create interface sets from shared entities
    result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::exchange_owned_mesh( std::vector< unsigned int >& exchange_procs,
                                             std::vector< Range* >& exchange_ents,
                                             std::vector< MPI_Request >& recv_ent_reqs,
                                             std::vector< MPI_Request >& recv_remoteh_reqs,
                                             const bool recv_posted,
                                             bool store_remote_handles,
                                             bool wait_all,
                                             bool migrate )
{
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting owned ents exchange." );
    }
#endif
 
    myDebug->tprintf( 1, "Entering exchange_owned_mesh\n" );
    if( myDebug->get_verbosity() == 4 )
    {
        msgs.clear();
        msgs.reserve( MAX_SHARING_PROCS );
    }
    unsigned int i;
    int ind, success;
    ErrorCode result = MB_SUCCESS;
    int incoming1 = 0, incoming2 = 0;
 
    // Set buffProcs with communicating procs
    unsigned int n_proc = exchange_procs.size();
    for( i = 0; i < n_proc; i++ )
    {
        ind    = get_buffers( exchange_procs[i] );
        result = add_verts( *exchange_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
 
        // Filter out entities already shared with destination
        Range tmp_range;
        result = filter_pstatus( *exchange_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
        if( !tmp_range.empty() )
        {
            *exchange_ents[i] = subtract( *exchange_ents[i], tmp_range );
        }
    }
 
    //===========================================
    // Post ghost irecv's for entities from all communicating procs
    //===========================================
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." );
    }
#endif
 
    // Index reqs the same as buffer/sharing procs indices
    if( !recv_posted )
    {
        reset_all_buffers();
        recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
        recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
        sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
 
        for( i = 0; i < n_proc; i++ )
        {
            ind = get_buffers( exchange_procs[i] );
            incoming1++;
            PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr,
                               INITIAL_BUFF_SIZE, MB_MESG_ENTS_SIZE, incoming1 );
            success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
                                 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
            if( success != MPI_SUCCESS )
            {
                MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
            }
        }
    }
    else
        incoming1 += n_proc;
 
    //===========================================
    // Get entities to be sent to neighbors
    // Need to get procs each entity is sent to
    //===========================================
    Range allsent, tmp_range;
    int dum_ack_buff;
    int npairs = 0;
    TupleList entprocs;
    for( i = 0; i < n_proc; i++ )
    {
        int n_ents = exchange_ents[i]->size();
        if( n_ents > 0 )
        {
            npairs += n_ents;  // Get the total # of proc/handle pairs
            allsent.merge( *exchange_ents[i] );
        }
    }
 
    // Allocate a TupleList of that size
    entprocs.initialize( 1, 0, 1, 0, npairs );
    entprocs.enableWriteAccess();
 
    // Put the proc/handle pairs in the list
    for( i = 0; i < n_proc; i++ )
    {
        for( Range::iterator rit = exchange_ents[i]->begin(); rit != exchange_ents[i]->end(); ++rit )
        {
            entprocs.vi_wr[entprocs.get_n()]  = exchange_procs[i];
            entprocs.vul_wr[entprocs.get_n()] = *rit;
            entprocs.inc_n();
        }
    }
 
    // Sort by handle
    moab::TupleList::buffer sort_buffer;
    sort_buffer.buffer_init( npairs );
    entprocs.sort( 1, &sort_buffer );
    sort_buffer.reset();
 
    myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
                      (unsigned long)allsent.size() );
 
    //===========================================
    // Pack and send ents from this proc to others
    //===========================================
    for( i = 0; i < n_proc; i++ )
    {
        ind = get_buffers( exchange_procs[i] );
        myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", exchange_ents[i]->compactness(),
                          (unsigned long)exchange_ents[i]->size() );
        // Reserve space on front for size and for initial buff size
        localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
        result = pack_buffer( *exchange_ents[i], false, true, store_remote_handles, buffProcs[ind],
                              localOwnedBuffs[ind], &entprocs, &allsent );
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *localOwnedBuffs[ind] );
        }
 
        // Send the buffer (size stored in front in send_buffer)
        result = send_buffer( exchange_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind],
                              recv_ent_reqs[3 * ind + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE,
                              ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recv_remoteh_reqs[3 * ind],
                              &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" );
    }
 
    entprocs.reset();
 
    //===========================================
    // Receive/unpack new entities
    //===========================================
    // Number of incoming messages is the number of procs we communicate with
    MPI_Status status;
    std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
    std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
    std::vector< std::vector< int > > L1p( buffProcs.size() );
    std::vector< EntityHandle > L2hloc, L2hrem;
    std::vector< unsigned int > L2p;
    std::vector< EntityHandle > new_ents;
 
    while( incoming1 )
    {
        // Wait for all recvs of ents before proceeding to sending remote handles,
        // b/c some procs may have sent to a 3rd proc ents owned by me;
        PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
 
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
        }
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming1--;
        bool done = false;
 
        // In case ind is for ack, we need index of one before it
        unsigned int base_ind = 3 * ( ind / 3 );
        result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1],
                              recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1],
                              sendReqs[base_ind + 2], done, ( store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ),
                              MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
 
        if( done )
        {
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] );
            }
 
            // Message completely received - process buffer that was sent
            remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
            result = unpack_buffer( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, buffProcs[ind / 3],
                                    ind / 3, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true );
            if( MB_SUCCESS != result )
            {
                std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
                print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false );
                return result;
            }
 
            if( recv_ent_reqs.size() != 3 * buffProcs.size() )
            {
                // Post irecv's for remote handles from new proc
                recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
                for( i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 )
                {
                    localOwnedBuffs[i / 3]->reset_buffer();
                    incoming2++;
                    PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr,
                                       INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
                    success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
                                         buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
                                         &recv_remoteh_reqs[i] );
                    if( success != MPI_SUCCESS )
                    {
                        MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
                    }
                }
                recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
                sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
            }
        }
    }
 
    // Assign and remove newly created elements from/to receive processor
    result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" );
    if( migrate )
    {
        result = remove_entities_part( allsent, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to remove entities to part" );
    }
 
    // Add requests for any new addl procs
    if( recv_ent_reqs.size() != 3 * buffProcs.size() )
    {
        // Shouldn't get here...
        MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" );
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." );
    }
#endif
 
    // we still need to wait on sendReqs, if they are not fulfilled yet
    if( wait_all )
    {
        if( myDebug->get_verbosity() == 5 )
        {
            success = MPI_Barrier( procConfig.proc_comm() );
        }
        else
        {
            MPI_Status mult_status[3 * MAX_SHARING_PROCS];
            success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
            if( MPI_SUCCESS != success )
            {
                MB_SET_ERR( MB_FAILURE, "Failed in waitall in exchange owned mesh" );
            }
        }
    }
 
    //===========================================
    // Send local handles for new entity to owner
    //===========================================
    for( i = 0; i < n_proc; i++ )
    {
        ind = get_buffers( exchange_procs[i] );
        // Reserve space on front for size and for initial buff size
        remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
 
        result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
        remoteOwnedBuffs[ind]->set_stored_size();
 
        if( myDebug->get_verbosity() == 4 )
        {
            msgs.resize( msgs.size() + 1 );
            msgs.back() = new Buffer( *remoteOwnedBuffs[ind] );
        }
        result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[3 * ind],
                              recv_remoteh_reqs[3 * ind + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
    }
 
    //===========================================
    // Process remote handles of my ghosteds
    //===========================================
    while( incoming2 )
    {
        PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
        }
 
        // OK, received something; decrement incoming counter
        incoming2--;
 
        PRINT_DEBUG_RECD( status );
 
        bool done             = false;
        unsigned int base_ind = 3 * ( ind / 3 );
        result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1],
                              recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3],
                              sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
 
        if( done )
        {
            // Incoming remote handles
            if( myDebug->get_verbosity() == 4 )
            {
                msgs.resize( msgs.size() + 1 );
                msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] );
            }
 
            localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
            result =
                unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
        }
    }
 
#ifdef MOAB_HAVE_MPE
    if( myDebug->get_verbosity() == 2 )
    {
        MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
        MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." );
    }
#endif
 
    //===========================================
    // Wait if requested
    //===========================================
    if( wait_all )
    {
        if( myDebug->get_verbosity() == 5 )
        {
            success = MPI_Barrier( procConfig.proc_comm() );
        }
        else
        {
            MPI_Status mult_status[3 * MAX_SHARING_PROCS];
            success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status );
            if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
        }
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitall in owned entity exchange" );
        }
    }
 
#ifndef NDEBUG
    result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" );
#endif
    myDebug->tprintf( 1, "Exiting exchange_owned_mesh\n" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_iface_entities( int other_proc, int dim, Range& iface_ents )
{
    Range iface_sets;
    ErrorCode result = MB_SUCCESS;
 
    for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
    {
        if( -1 != other_proc && !is_iface_proc( *rit, other_proc ) ) continue;
 
        if( -1 == dim )
        {
            result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" );
        }
        else
        {
            result = mbImpl->get_entities_by_dimension( *rit, dim, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::assign_entities_part( std::vector< EntityHandle >& entities, const int proc )
{
    EntityHandle part_set;
    ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" );
 
    if( part_set > 0 )
    {
        result = mbImpl->add_entities( part_set, &entities[0], entities.size() );MB_CHK_SET_ERR( result, "Failed to add entities to part set" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::remove_entities_part( Range& entities, const int proc )
{
    EntityHandle part_set;
    ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" );
 
    if( part_set > 0 )
    {
        result = mbImpl->remove_entities( part_set, entities );MB_CHK_SET_ERR( result, "Failed to remove entities from part set" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::check_sent_ents( Range& allsent )
{
    // Check entities to make sure there are no zero-valued remote handles
    // where they shouldn't be
    std::vector< unsigned char > pstat( allsent.size() );
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), allsent, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
    std::vector< EntityHandle > handles( allsent.size() );
    result = mbImpl->tag_get_data( sharedh_tag(), allsent, &handles[0] );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
    std::vector< int > procs( allsent.size() );
    result = mbImpl->tag_get_data( sharedp_tag(), allsent, &procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
 
    Range bad_entities;
 
    Range::iterator rit;
    unsigned int i;
    EntityHandle dum_hs[MAX_SHARING_PROCS];
    int dum_ps[MAX_SHARING_PROCS];
 
    for( rit = allsent.begin(), i = 0; rit != allsent.end(); ++rit, i++ )
    {
        if( -1 != procs[i] && 0 == handles[i] )
            bad_entities.insert( *rit );
        else
        {
            // Might be multi-shared...
            result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, dum_ps );
            if( MB_TAG_NOT_FOUND == result )
                continue;
            else if( MB_SUCCESS != result )
                MB_SET_ERR( result, "Failed to get sharedps tag data" );
            result = mbImpl->tag_get_data( sharedhs_tag(), &( *rit ), 1, dum_hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
 
            // Find first non-set proc
            int* ns_proc  = std::find( dum_ps, dum_ps + MAX_SHARING_PROCS, -1 );
            int num_procs = ns_proc - dum_ps;
            assert( num_procs <= MAX_SHARING_PROCS );
            // Now look for zero handles in active part of dum_hs
            EntityHandle* ns_handle = std::find( dum_hs, dum_hs + num_procs, 0 );
            int num_handles         = ns_handle - dum_hs;
            assert( num_handles <= num_procs );
            if( num_handles != num_procs ) bad_entities.insert( *rit );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::pack_remote_handles( std::vector< EntityHandle >& L1hloc,
                                             std::vector< EntityHandle >& L1hrem,
                                             std::vector< int >& L1p,
                                             unsigned int /*to_proc*/,
                                             Buffer* buff )
{
    assert( std::find( L1hloc.begin(), L1hloc.end(), (EntityHandle)0 ) == L1hloc.end() );
 
    // 2 vectors of handles plus ints
    buff->check_space( ( ( L1p.size() + 1 ) * sizeof( int ) + ( L1hloc.size() + 1 ) * sizeof( EntityHandle ) +
                         ( L1hrem.size() + 1 ) * sizeof( EntityHandle ) ) );
 
    // Should be in pairs of handles
    PACK_INT( buff->buff_ptr, L1hloc.size() );
    PACK_INTS( buff->buff_ptr, &L1p[0], L1p.size() );
    // Pack handles in reverse order, (remote, local), so on destination they
    // are ordered (local, remote)
    PACK_EH( buff->buff_ptr, &L1hrem[0], L1hrem.size() );
    PACK_EH( buff->buff_ptr, &L1hloc[0], L1hloc.size() );
 
    buff->set_stored_size();
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::unpack_remote_handles( unsigned int from_proc,
                                               unsigned char*& buff_ptr,
                                               std::vector< EntityHandle >& L2hloc,
                                               std::vector< EntityHandle >& L2hrem,
                                               std::vector< unsigned int >& L2p )
{
    // Incoming remote handles; use to set remote handles
    int num_eh;
    UNPACK_INT( buff_ptr, num_eh );
 
    unsigned char* buff_proc = buff_ptr;
    buff_ptr += num_eh * sizeof( int );
    unsigned char* buff_rem = buff_ptr + num_eh * sizeof( EntityHandle );
    ErrorCode result;
    EntityHandle hpair[2], new_h;
    int proc;
    for( int i = 0; i < num_eh; i++ )
    {
        UNPACK_INT( buff_proc, proc );
        // Handles packed (local, remote), though here local is either on this
        // proc or owner proc, depending on value of proc (-1 = here, otherwise owner);
        // this is decoded in find_existing_entity
        UNPACK_EH( buff_ptr, hpair, 1 );
        UNPACK_EH( buff_rem, hpair + 1, 1 );
 
        if( -1 != proc )
        {
            result = find_existing_entity( false, proc, hpair[0], 3, NULL, 0, mbImpl->type_from_handle( hpair[1] ),
                                           L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Didn't get existing entity" );
            if( new_h )
                hpair[0] = new_h;
            else
                hpair[0] = 0;
        }
        if( !( hpair[0] && hpair[1] ) ) return MB_FAILURE;
        int this_proc = from_proc;
        result        = update_remote_data( hpair[0], &this_proc, hpair + 1, 1, 0 );MB_CHK_SET_ERR( result, "Failed to set remote data range on sent entities in ghost exchange" );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_ghosted_entities( int bridge_dim,
                                              int ghost_dim,
                                              int to_proc,
                                              int num_layers,
                                              int addl_ents,
                                              Range& ghosted_ents )
{
    // Get bridge ents on interface(s)
    Range from_ents;
    ErrorCode result = MB_SUCCESS;
    assert( 0 < num_layers );
    for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
    {
        if( !is_iface_proc( *rit, to_proc ) ) continue;
 
        // Get starting "from" entities
        if( bridge_dim == -1 )
        {
            result = mbImpl->get_entities_by_handle( *rit, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" );
        }
        else
        {
            result = mbImpl->get_entities_by_dimension( *rit, bridge_dim, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" );
        }
 
        // Need to get layers of bridge-adj entities
        if( from_ents.empty() ) continue;
        result =
            MeshTopoUtil( mbImpl ).get_bridge_adjacencies( from_ents, bridge_dim, ghost_dim, ghosted_ents, num_layers );MB_CHK_SET_ERR( result, "Failed to get bridge adjacencies" );
    }
 
    result = add_verts( ghosted_ents );MB_CHK_SET_ERR( result, "Failed to add verts" );
 
    if( addl_ents )
    {
        // First get the ents of ghost_dim
        Range tmp_ents, tmp_owned, tmp_notowned;
        tmp_owned = ghosted_ents.subset_by_dimension( ghost_dim );
        if( tmp_owned.empty() ) return result;
 
        tmp_notowned = tmp_owned;
 
        // Next, filter by pstatus; can only create adj entities for entities I own
        result = filter_pstatus( tmp_owned, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &tmp_owned );MB_CHK_SET_ERR( result, "Failed to filter owned entities" );
 
        tmp_notowned -= tmp_owned;
 
        // Get edges first
        if( 1 == addl_ents || 3 == addl_ents )
        {
            result = mbImpl->get_adjacencies( tmp_owned, 1, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for owned ghost entities" );
            result = mbImpl->get_adjacencies( tmp_notowned, 1, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for notowned ghost entities" );
        }
        if( 2 == addl_ents || 3 == addl_ents )
        {
            result = mbImpl->get_adjacencies( tmp_owned, 2, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for owned ghost entities" );
            result = mbImpl->get_adjacencies( tmp_notowned, 2, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for notowned ghost entities" );
        }
 
        ghosted_ents.merge( tmp_ents );
    }
 
    return result;
}
 
ErrorCode ParallelComm::add_verts( Range& sent_ents )
{
    // Get the verts adj to these entities, since we'll have to send those too
 
    // First check sets
    std::pair< Range::const_iterator, Range::const_iterator > set_range = sent_ents.equal_range( MBENTITYSET );
    ErrorCode result                                                    = MB_SUCCESS, tmp_result;
    for( Range::const_iterator rit = set_range.first; rit != set_range.second; ++rit )
    {
        tmp_result = mbImpl->get_entities_by_type( *rit, MBVERTEX, sent_ents );MB_CHK_SET_ERR( tmp_result, "Failed to get contained verts" );
    }
 
    // Now non-sets
    Range tmp_ents;
    std::copy( sent_ents.begin(), set_range.first, range_inserter( tmp_ents ) );
    result = mbImpl->get_adjacencies( tmp_ents, 0, false, sent_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get vertices adj to ghosted ents" );
 
    // if polyhedra, need to add all faces from there
    Range polyhedra = sent_ents.subset_by_type( MBPOLYHEDRON );
    // get all faces adjacent to every polyhedra
    result = mbImpl->get_connectivity( polyhedra, sent_ents );MB_CHK_SET_ERR( result, "Failed to get polyhedra faces" );
    return result;
}
 
ErrorCode ParallelComm::exchange_tags( const std::vector< Tag >& src_tags,
                                       const std::vector< Tag >& dst_tags,
                                       const Range& entities_in )
{
    ErrorCode result;
    int success;
 
    myDebug->tprintf( 1, "Entering exchange_tags\n" );
 
    // Get all procs interfacing to this proc
    std::set< unsigned int > exch_procs;
    result = get_comm_procs( exch_procs );
 
    // Post ghost irecv's for all interface procs
    // Index requests the same as buffer/sharing procs indices
    std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    // sent_ack_reqs(buffProcs.size(), MPI_REQUEST_NULL);
    std::vector< unsigned int >::iterator sit;
    int ind;
 
    reset_all_buffers();
    int incoming = 0;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        incoming++;
        PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_TAGS_SIZE, incoming );
 
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
                             MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
        }
    }
 
    // Pack and send tags from this proc to others
    // Make sendReqs vector to simplify initialization
    sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
 
    // Take all shared entities if incoming list is empty
    Range entities;
    if( entities_in.empty() )
        std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
    else
        entities = entities_in;
 
    int dum_ack_buff;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        Range tag_ents = entities;
 
        // Get ents shared by proc *sit
        result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
 
        // Remote nonowned entities
        if( !tag_ents.empty() )
        {
            result = filter_pstatus( tag_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed pstatus NOT check" );
        }
 
        // Pack-send; this also posts receives if store_remote_handles is true
        std::vector< Range > tag_ranges;
        for( std::vector< Tag >::const_iterator vit = src_tags.begin(); vit != src_tags.end(); ++vit )
        {
            const void* ptr;
            int sz;
            if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS )
            {
                Range tagged_ents;
                mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents );
                tag_ranges.push_back( intersect( tag_ents, tagged_ents ) );
            }
            else
            {
                tag_ranges.push_back( tag_ents );
            }
        }
 
        // Pack the data
        // Reserve space on front for size and for initial buff size
        localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
 
        result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" );
 
        // Now send it
        result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
                              recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
    }
 
    // Receive/unpack tags
    while( incoming )
    {
        MPI_Status status;
        int index_in_recv_requests;
        PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in tag exchange" );
        }
        // Processor index in the list is divided by 3
        ind = index_in_recv_requests / 3;
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming--;
 
        bool done = false;
        std::vector< EntityHandle > dum_vec;
        result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
                              recv_tag_reqs[3 * ind + 1],  // This is for receiving the second message
                              recv_tag_reqs[3 * ind + 2],  // This would be for ack, but it is not
                                                           // used; consider removing it
                              incoming, localOwnedBuffs[ind],
                              sendReqs[3 * ind + 1],  // Send request for sending the second message
                              sendReqs[3 * ind + 2],  // This is for sending the ack
                              done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
        if( done )
        {
            remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind] );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" );
        }
    }
 
    // OK, now wait
    if( myDebug->get_verbosity() == 5 )
    {
        success = MPI_Barrier( procConfig.proc_comm() );
    }
    else
    {
        MPI_Status status[3 * MAX_SHARING_PROCS];
        success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
    }
    if( MPI_SUCCESS != success )
    {
        MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
    }
 
    // If source tag is not equal to destination tag, then
    // do local copy for owned entities (communicate w/ self)
    assert( src_tags.size() == dst_tags.size() );
    if( src_tags != dst_tags )
    {
        std::vector< unsigned char > data;
        Range owned_ents;
        if( entities_in.empty() )
            std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
        else
            owned_ents = entities_in;
        result = filter_pstatus( owned_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failure to get subset of owned entities" );
 
        if( !owned_ents.empty() )
        {  // Check this here, otherwise we get
            // Unexpected results from get_entities_by_type_and_tag w/ Interface::INTERSECT
            for( size_t i = 0; i < src_tags.size(); i++ )
            {
                if( src_tags[i] == dst_tags[i] ) continue;
 
                Range tagged_ents( owned_ents );
                result = mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &src_tags[0], 0, 1, tagged_ents,
                                                               Interface::INTERSECT );MB_CHK_SET_ERR( result, "get_entities_by_type_and_tag(type == MBMAXTYPE) failed" );
 
                int sz, size2;
                result = mbImpl->tag_get_bytes( src_tags[i], sz );MB_CHK_SET_ERR( result, "tag_get_size failed" );
                result = mbImpl->tag_get_bytes( dst_tags[i], size2 );MB_CHK_SET_ERR( result, "tag_get_size failed" );
                if( sz != size2 )
                {
                    MB_SET_ERR( MB_FAILURE, "tag sizes don't match" );
                }
 
                data.resize( sz * tagged_ents.size() );
                result = mbImpl->tag_get_data( src_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_get_data failed" );
                result = mbImpl->tag_set_data( dst_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_set_data failed" );
            }
        }
    }
 
    myDebug->tprintf( 1, "Exiting exchange_tags" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::reduce_tags( const std::vector< Tag >& src_tags,
                                     const std::vector< Tag >& dst_tags,
                                     const MPI_Op mpi_op,
                                     const Range& entities_in )
{
    ErrorCode result;
    int success;
 
    myDebug->tprintf( 1, "Entering reduce_tags\n" );
 
    // Check that restrictions are met: number of source/dst tags...
    if( src_tags.size() != dst_tags.size() )
    {
        MB_SET_ERR( MB_FAILURE, "Source and destination tag handles must be specified for reduce_tags" );
    }
 
    // ... tag data types
    std::vector< Tag >::const_iterator vits, vitd;
    int tags_size, tagd_size;
    DataType tags_type, tagd_type;
    std::vector< unsigned char > vals;
    std::vector< int > tags_sizes;
    for( vits = src_tags.begin(), vitd = dst_tags.begin(); vits != src_tags.end(); ++vits, ++vitd )
    {
        // Checks on tag characteristics
        result = mbImpl->tag_get_data_type( *vits, tags_type );MB_CHK_SET_ERR( result, "Failed to get src tag data type" );
        if( tags_type != MB_TYPE_INTEGER && tags_type != MB_TYPE_DOUBLE && tags_type != MB_TYPE_BIT )
        {
            MB_SET_ERR( MB_FAILURE, "Src/dst tags must have integer, double, or bit data type" );
        }
 
        result = mbImpl->tag_get_bytes( *vits, tags_size );MB_CHK_SET_ERR( result, "Failed to get src tag bytes" );
        vals.resize( tags_size );
        result = mbImpl->tag_get_default_value( *vits, &vals[0] );MB_CHK_SET_ERR( result, "Src tag must have default value" );
 
        tags_sizes.push_back( tags_size );
 
        // OK, those passed; now check whether dest tags, if specified, agree with src tags
        if( *vits == *vitd ) continue;
 
        result = mbImpl->tag_get_bytes( *vitd, tagd_size );MB_CHK_SET_ERR( result, "Coudln't get dst tag bytes" );
        if( tags_size != tagd_size )
        {
            MB_SET_ERR( MB_FAILURE, "Sizes between src and dst tags don't match" );
        }
        result = mbImpl->tag_get_data_type( *vitd, tagd_type );MB_CHK_SET_ERR( result, "Coudln't get dst tag data type" );
        if( tags_type != tagd_type )
        {
            MB_SET_ERR( MB_FAILURE, "Src and dst tags must be of same data type" );
        }
    }
 
    // Get all procs interfacing to this proc
    std::set< unsigned int > exch_procs;
    result = get_comm_procs( exch_procs );
 
    // Post ghost irecv's for all interface procs
    // Index requests the same as buffer/sharing procs indices
    std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
 
    std::vector< unsigned int >::iterator sit;
    int ind;
 
    reset_all_buffers();
    int incoming = 0;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        incoming++;
        PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_TAGS_SIZE, incoming );
 
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
                             MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
        }
    }
 
    // Pack and send tags from this proc to others
    // Make sendReqs vector to simplify initialization
    sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
 
    // Take all shared entities if incoming list is empty
    Range entities;
    if( entities_in.empty() )
        std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
    else
        entities = entities_in;
 
    // If the tags are different, copy the source to the dest tag locally
    std::vector< Tag >::const_iterator vit = src_tags.begin(), vit2 = dst_tags.begin();
    std::vector< int >::const_iterator vsizes = tags_sizes.begin();
    for( ; vit != src_tags.end(); ++vit, ++vit2, ++vsizes )
    {
        if( *vit == *vit2 ) continue;
        vals.resize( entities.size() * ( *vsizes ) );
        result = mbImpl->tag_get_data( *vit, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't get data properly" );
        result = mbImpl->tag_set_data( *vit2, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't set data properly" );
    }
 
    int dum_ack_buff;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        Range tag_ents = entities;
 
        // Get ents shared by proc *sit
        result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
 
        // Pack-send
        std::vector< Range > tag_ranges;
        for( vit = src_tags.begin(); vit != src_tags.end(); ++vit )
        {
            const void* ptr;
            int sz;
            if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS )
            {
                Range tagged_ents;
                mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents );
                tag_ranges.push_back( intersect( tag_ents, tagged_ents ) );
            }
            else
                tag_ranges.push_back( tag_ents );
        }
 
        // Pack the data
        // Reserve space on front for size and for initial buff size
        localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
 
        result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" );
 
        // Now send it
        result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
                              recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
    }
 
    // Receive/unpack tags
    while( incoming )
    {
        MPI_Status status;
        int index_in_recv_requests;
        PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
        }
        ind = index_in_recv_requests / 3;
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming--;
 
        bool done = false;
        std::vector< EntityHandle > dum_vec;
        result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
                              recv_tag_reqs[3 * ind + 1],  // This is for receiving the second message
                              recv_tag_reqs[3 * ind + 2],  // This would be for ack, but it is not
                                                           // used; consider removing it
                              incoming, localOwnedBuffs[ind],
                              sendReqs[3 * ind + 1],  // Send request for sending the second message
                              sendReqs[3 * ind + 2],  // This is for sending the ack
                              done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
        if( done )
        {
            remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
            result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind], &mpi_op );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" );
        }
    }
 
    // OK, now wait
    if( myDebug->get_verbosity() == 5 )
    {
        success = MPI_Barrier( procConfig.proc_comm() );
    }
    else
    {
        MPI_Status status[3 * MAX_SHARING_PROCS];
        success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
    }
    if( MPI_SUCCESS != success )
    {
        MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
    }
 
    myDebug->tprintf( 1, "Exiting reduce_tags" );
 
    return MB_SUCCESS;
}
 
//! return sharedp tag
Tag ParallelComm::sharedp_tag()
{
    if( !sharedpTag )
    {
        int def_val      = -1;
        ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROC_TAG_NAME, 1, MB_TYPE_INTEGER, sharedpTag,
                                                   MB_TAG_DENSE | MB_TAG_CREAT, &def_val );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return sharedpTag;
}
 
//! return sharedps tag
Tag ParallelComm::sharedps_tag()
{
    if( !sharedpsTag )
    {
        ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROCS_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_INTEGER,
                                                   sharedpsTag, MB_TAG_SPARSE | MB_TAG_CREAT );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return sharedpsTag;
}
 
//! return sharedh tag
Tag ParallelComm::sharedh_tag()
{
    if( !sharedhTag )
    {
        EntityHandle def_val = 0;
        ErrorCode result     = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLE_TAG_NAME, 1, MB_TYPE_HANDLE, sharedhTag,
                                                       MB_TAG_DENSE | MB_TAG_CREAT, &def_val );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return sharedhTag;
}
 
//! return sharedhs tag
Tag ParallelComm::sharedhs_tag()
{
    if( !sharedhsTag )
    {
        ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLES_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_HANDLE,
                                                   sharedhsTag, MB_TAG_SPARSE | MB_TAG_CREAT );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return sharedhsTag;
}
 
//! return pstatus tag
Tag ParallelComm::pstatus_tag()
{
    if( !pstatusTag )
    {
        unsigned char tmp_pstatus = 0;
        ErrorCode result          = mbImpl->tag_get_handle( PARALLEL_STATUS_TAG_NAME, 1, MB_TYPE_OPAQUE, pstatusTag,
                                                            MB_TAG_DENSE | MB_TAG_CREAT, &tmp_pstatus );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return pstatusTag;
}
 
//! return partition set tag
Tag ParallelComm::partition_tag()
{
    if( !partitionTag )
    {
        int dum_id       = -1;
        ErrorCode result = mbImpl->tag_get_handle( PARALLEL_PARTITION_TAG_NAME, 1, MB_TYPE_INTEGER, partitionTag,
                                                   MB_TAG_SPARSE | MB_TAG_CREAT, &dum_id );
        if( MB_SUCCESS != result ) return 0;
    }
 
    return partitionTag;
}
 
//! return pcomm tag; passes in impl 'cuz this is a static function
Tag ParallelComm::pcomm_tag( Interface* impl, bool create_if_missing )
{
    Tag this_tag = 0;
    ErrorCode result;
    if( create_if_missing )
    {
        result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ),
                                       MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE | MB_TAG_CREAT );
    }
    else
    {
        result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ),
                                       MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE );
    }
 
    if( MB_SUCCESS != result ) return 0;
 
    return this_tag;
}
 
//! get the indexed pcomm object from the interface
ParallelComm* ParallelComm::get_pcomm( Interface* impl, const int index )
{
    Tag pc_tag = pcomm_tag( impl, false );
    if( 0 == pc_tag ) return NULL;
 
    const EntityHandle root = 0;
    ParallelComm* pc_array[MAX_SHARING_PROCS];
    ErrorCode result = impl->tag_get_data( pc_tag, &root, 1, (void*)pc_array );
    if( MB_SUCCESS != result ) return NULL;
 
    return pc_array[index];
}
 
ErrorCode ParallelComm::get_all_pcomm( Interface* impl, std::vector< ParallelComm* >& list )
{
    Tag pc_tag = pcomm_tag( impl, false );
    if( 0 == pc_tag ) return MB_TAG_NOT_FOUND;
 
    const EntityHandle root = 0;
    ParallelComm* pc_array[MAX_SHARING_PROCS];
    ErrorCode rval = impl->tag_get_data( pc_tag, &root, 1, pc_array );
    if( MB_SUCCESS != rval ) return rval;
 
    for( int i = 0; i < MAX_SHARING_PROCS; i++ )
    {
        if( pc_array[i] ) list.push_back( pc_array[i] );
    }
 
    return MB_SUCCESS;
}
 
//! get the indexed pcomm object from the interface
ParallelComm* ParallelComm::get_pcomm( Interface* impl, EntityHandle prtn, const MPI_Comm* comm )
{
    ErrorCode rval;
    ParallelComm* result = 0;
 
    Tag prtn_tag;
    rval =
        impl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag, MB_TAG_SPARSE | MB_TAG_CREAT );
    if( MB_SUCCESS != rval ) return 0;
 
    int pcomm_id;
    rval = impl->tag_get_data( prtn_tag, &prtn, 1, &pcomm_id );
    if( MB_SUCCESS == rval )
    {
        result = get_pcomm( impl, pcomm_id );
    }
    else if( MB_TAG_NOT_FOUND == rval && comm )
    {
        result = new ParallelComm( impl, *comm, &pcomm_id );
        if( !result ) return 0;
        result->set_partitioning( prtn );
 
        rval = impl->tag_set_data( prtn_tag, &prtn, 1, &pcomm_id );
        if( MB_SUCCESS != rval )
        {
            delete result;
            result = 0;
        }
    }
 
    return result;
}
 
ErrorCode ParallelComm::set_partitioning( EntityHandle set )
{
    ErrorCode rval;
    Tag prtn_tag;
    rval = mbImpl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag,
                                   MB_TAG_SPARSE | MB_TAG_CREAT );
    if( MB_SUCCESS != rval ) return rval;
 
    // Get my id
    ParallelComm* pcomm_arr[MAX_SHARING_PROCS];
    Tag pc_tag = pcomm_tag( mbImpl, false );
    if( 0 == pc_tag ) return MB_FAILURE;
    const EntityHandle root = 0;
    ErrorCode result        = mbImpl->tag_get_data( pc_tag, &root, 1, pcomm_arr );
    if( MB_SUCCESS != result ) return MB_FAILURE;
    int id = std::find( pcomm_arr, pcomm_arr + MAX_SHARING_PROCS, this ) - pcomm_arr;
    if( id == MAX_SHARING_PROCS ) return MB_FAILURE;
 
    EntityHandle old = partitioningSet;
    if( old )
    {
        rval = mbImpl->tag_delete_data( prtn_tag, &old, 1 );
        if( MB_SUCCESS != rval ) return rval;
        partitioningSet = 0;
    }
 
    if( !set ) return MB_SUCCESS;
 
    Range contents;
    if( old )
    {
        rval = mbImpl->get_entities_by_handle( old, contents );
        if( MB_SUCCESS != rval ) return rval;
    }
    else
    {
        contents = partition_sets();
    }
 
    rval = mbImpl->add_entities( set, contents );
    if( MB_SUCCESS != rval ) return rval;
 
    // Store pcomm id on new partition set
    rval = mbImpl->tag_set_data( prtn_tag, &set, 1, &id );
    if( MB_SUCCESS != rval ) return rval;
 
    partitioningSet = set;
    return MB_SUCCESS;
}
 
//! return all the entities in parts owned locally
ErrorCode ParallelComm::get_part_entities( Range& ents, int dim )
{
    ErrorCode result;
 
    for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit )
    {
        Range tmp_ents;
        if( -1 == dim )
            result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true );
        else
            result = mbImpl->get_entities_by_dimension( *rit, dim, tmp_ents, true );
 
        if( MB_SUCCESS != result ) return result;
        ents.merge( tmp_ents );
    }
 
    return MB_SUCCESS;
}
 
/** \brief Return the rank of the entity owner
 */
ErrorCode ParallelComm::get_owner_handle( EntityHandle entity, int& owner, EntityHandle& handle )
{
    unsigned char pstat;
    int sharing_procs[MAX_SHARING_PROCS];
    EntityHandle sharing_handles[MAX_SHARING_PROCS];
 
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
    if( !( pstat & PSTATUS_NOT_OWNED ) )
    {
        owner  = proc_config().proc_rank();
        handle = entity;
    }
    else if( pstat & PSTATUS_MULTISHARED )
    {
        result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
        owner  = sharing_procs[0];
        result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
        handle = sharing_handles[0];
    }
    else if( pstat & PSTATUS_SHARED )
    {
        result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
        owner  = sharing_procs[0];
        result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
        handle = sharing_handles[0];
    }
    else
    {
        owner  = -1;
        handle = 0;
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_global_part_count( int& count_out ) const
{
    count_out = globalPartCount;
    return count_out < 0 ? MB_FAILURE : MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_part_owner( int part_id, int& owner ) const
{
    // FIXME: assumes only 1 local part
    owner = part_id;
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_part_id( EntityHandle /*part*/, int& id_out ) const
{
    // FIXME: assumes only 1 local part
    id_out = proc_config().proc_rank();
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_part_handle( int id, EntityHandle& handle_out ) const
{
    // FIXME: assumes only 1 local part
    if( (unsigned)id != proc_config().proc_rank() ) return MB_ENTITY_NOT_FOUND;
    handle_out = partition_sets().front();
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::create_part( EntityHandle& set_out )
{
    // Mark as invalid so we know that it needs to be updated
    globalPartCount = -1;
 
    // Create set representing part
    ErrorCode rval = mbImpl->create_meshset( MESHSET_SET, set_out );
    if( MB_SUCCESS != rval ) return rval;
 
    // Set tag on set
    int val = proc_config().proc_rank();
    rval    = mbImpl->tag_set_data( part_tag(), &set_out, 1, &val );
 
    if( MB_SUCCESS != rval )
    {
        mbImpl->delete_entities( &set_out, 1 );
        return rval;
    }
 
    if( get_partitioning() )
    {
        rval = mbImpl->add_entities( get_partitioning(), &set_out, 1 );
        if( MB_SUCCESS != rval )
        {
            mbImpl->delete_entities( &set_out, 1 );
            return rval;
        }
    }
 
    moab::Range& pSets = this->partition_sets();
    if( pSets.index( set_out ) < 0 )
    {
        pSets.insert( set_out );
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::destroy_part( EntityHandle part_id )
{
    // Mark as invalid so we know that it needs to be updated
    globalPartCount = -1;
 
    ErrorCode rval;
    if( get_partitioning() )
    {
        rval = mbImpl->remove_entities( get_partitioning(), &part_id, 1 );
        if( MB_SUCCESS != rval ) return rval;
    }
 
    moab::Range& pSets = this->partition_sets();
    if( pSets.index( part_id ) >= 0 )
    {
        pSets.erase( part_id );
    }
    return mbImpl->delete_entities( &part_id, 1 );
}
 
ErrorCode ParallelComm::collective_sync_partition()
{
    int count       = partition_sets().size();
    globalPartCount = 0;
    int err         = MPI_Allreduce( &count, &globalPartCount, 1, MPI_INT, MPI_SUM, proc_config().proc_comm() );
    return err ? MB_FAILURE : MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_part_neighbor_ids( EntityHandle part,
                                               int neighbors_out[MAX_SHARING_PROCS],
                                               int& num_neighbors_out )
{
    ErrorCode rval;
    Range iface;
    rval = get_interface_sets( part, iface );
    if( MB_SUCCESS != rval ) return rval;
 
    num_neighbors_out = 0;
    int n, j = 0;
    int tmp[MAX_SHARING_PROCS] = { 0 }, curr[MAX_SHARING_PROCS] = { 0 };
    int* parts[2] = { neighbors_out, tmp };
    for( Range::iterator i = iface.begin(); i != iface.end(); ++i )
    {
        unsigned char pstat;
        rval = get_sharing_data( *i, curr, NULL, pstat, n );
        if( MB_SUCCESS != rval ) return rval;
        std::sort( curr, curr + n );
        assert( num_neighbors_out < MAX_SHARING_PROCS );
        int* k            = std::set_union( parts[j], parts[j] + num_neighbors_out, curr, curr + n, parts[1 - j] );
        j                 = 1 - j;
        num_neighbors_out = k - parts[j];
    }
    if( parts[j] != neighbors_out ) std::copy( parts[j], parts[j] + num_neighbors_out, neighbors_out );
 
    // Remove input part from list
    int id;
    rval = get_part_id( part, id );
    if( MB_SUCCESS == rval )
        num_neighbors_out = std::remove( neighbors_out, neighbors_out + num_neighbors_out, id ) - neighbors_out;
    return rval;
}
 
ErrorCode ParallelComm::get_interface_sets( EntityHandle, Range& iface_sets_out, int* adj_part_id )
{
    // FIXME : assumes one part per processor.
    // Need to store part iface sets as children to implement
    // this correctly.
    iface_sets_out = interface_sets();
 
    if( adj_part_id )
    {
        int part_ids[MAX_SHARING_PROCS], num_parts;
        Range::iterator i = iface_sets_out.begin();
        while( i != iface_sets_out.end() )
        {
            unsigned char pstat;
            ErrorCode rval = get_sharing_data( *i, part_ids, NULL, pstat, num_parts );
            if( MB_SUCCESS != rval ) return rval;
 
            if( std::find( part_ids, part_ids + num_parts, *adj_part_id ) - part_ids != num_parts )
                ++i;
            else
                i = iface_sets_out.erase( i );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_owning_part( EntityHandle handle, int& owning_part_id, EntityHandle* remote_handle )
{
    // FIXME : assumes one part per proc, and therefore part_id == rank
 
    // If entity is not shared, then we're the owner.
    unsigned char pstat;
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &handle, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
    if( !( pstat & PSTATUS_NOT_OWNED ) )
    {
        owning_part_id = proc_config().proc_rank();
        if( remote_handle ) *remote_handle = handle;
        return MB_SUCCESS;
    }
 
    // If entity is shared with one other proc, then
    // sharedp_tag will contain a positive value.
    result = mbImpl->tag_get_data( sharedp_tag(), &handle, 1, &owning_part_id );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
    if( owning_part_id != -1 )
    {
        // Done?
        if( !remote_handle ) return MB_SUCCESS;
 
        // Get handles on remote processors (and this one)
        return mbImpl->tag_get_data( sharedh_tag(), &handle, 1, remote_handle );
    }
 
    // If here, then the entity is shared with at least two other processors.
    // Get the list from the sharedps_tag
    const void* part_id_list = 0;
    result                   = mbImpl->tag_get_by_ptr( sharedps_tag(), &handle, 1, &part_id_list );
    if( MB_SUCCESS != result ) return result;
    owning_part_id = ( (const int*)part_id_list )[0];
 
    // Done?
    if( !remote_handle ) return MB_SUCCESS;
 
    // Get remote handles
    const void* handle_list = 0;
    result                  = mbImpl->tag_get_by_ptr( sharedhs_tag(), &handle, 1, &handle_list );
    if( MB_SUCCESS != result ) return result;
 
    *remote_handle = ( (const EntityHandle*)handle_list )[0];
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_sharing_parts( EntityHandle entity,
                                           int part_ids_out[MAX_SHARING_PROCS],
                                           int& num_part_ids_out,
                                           EntityHandle remote_handles[MAX_SHARING_PROCS] )
{
    // FIXME : assumes one part per proc, and therefore part_id == rank
 
    // If entity is not shared, then we're the owner.
    unsigned char pstat;
    ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
    if( !( pstat & PSTATUS_SHARED ) )
    {
        part_ids_out[0] = proc_config().proc_rank();
        if( remote_handles ) remote_handles[0] = entity;
        num_part_ids_out = 1;
        return MB_SUCCESS;
    }
 
    // If entity is shared with one other proc, then
    // sharedp_tag will contain a positive value.
    result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, part_ids_out );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
    if( part_ids_out[0] != -1 )
    {
        num_part_ids_out = 2;
        part_ids_out[1]  = proc_config().proc_rank();
 
        // Done?
        if( !remote_handles ) return MB_SUCCESS;
 
        // Get handles on remote processors (and this one)
        remote_handles[1] = entity;
        return mbImpl->tag_get_data( sharedh_tag(), &entity, 1, remote_handles );
    }
 
    // If here, then the entity is shared with at least two other processors.
    // Get the list from the sharedps_tag
    result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, part_ids_out );
    if( MB_SUCCESS != result ) return result;
    // Count number of valid (positive) entries in sharedps_tag
    for( num_part_ids_out = 0; num_part_ids_out < MAX_SHARING_PROCS && part_ids_out[num_part_ids_out] >= 0;
         num_part_ids_out++ )
        ;
        // part_ids_out[num_part_ids_out++] = proc_config().proc_rank();
#ifndef NDEBUG
    int my_idx = std::find( part_ids_out, part_ids_out + num_part_ids_out, proc_config().proc_rank() ) - part_ids_out;
    assert( my_idx < num_part_ids_out );
#endif
 
    // Done?
    if( !remote_handles ) return MB_SUCCESS;
 
    // Get remote handles
    result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, remote_handles );
    // remote_handles[num_part_ids_out - 1] = entity;
    assert( remote_handles[my_idx] == entity );
 
    return result;
}
 
ErrorCode ParallelComm::pack_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data )
{
    // Build up send buffers
    ErrorCode rval = MB_SUCCESS;
    int ent_procs[MAX_SHARING_PROCS];
    EntityHandle handles[MAX_SHARING_PROCS];
    int num_sharing, tmp_int;
    SharedEntityData tmp;
    send_data.resize( buffProcs.size() );
    for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i )
    {
        tmp.remote = *i;  // Swap local/remote so they're correct on the remote proc.
        rval       = get_owner( *i, tmp_int );
        tmp.owner  = tmp_int;
        if( MB_SUCCESS != rval ) return rval;
 
        unsigned char pstat;
        rval = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing );
        if( MB_SUCCESS != rval ) return rval;
        for( int j = 0; j < num_sharing; j++ )
        {
            if( ent_procs[j] == (int)proc_config().proc_rank() ) continue;
            tmp.local = handles[j];
            int ind   = get_buffers( ent_procs[j] );
            assert( -1 != ind );
            if( (int)send_data.size() < ind + 1 ) send_data.resize( ind + 1 );
            send_data[ind].push_back( tmp );
        }
    }
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::exchange_all_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data,
                                                     std::vector< std::vector< SharedEntityData > >& result )
{
    int ierr;
    const int tag      = 0;
    const MPI_Comm cm  = procConfig.proc_comm();
    const int num_proc = buffProcs.size();
    const std::vector< int > procs( buffProcs.begin(), buffProcs.end() );
    std::vector< MPI_Request > recv_req( buffProcs.size(), MPI_REQUEST_NULL );
    std::vector< MPI_Request > send_req( buffProcs.size(), MPI_REQUEST_NULL );
 
    // Set up to receive sizes
    std::vector< int > sizes_send( num_proc ), sizes_recv( num_proc );
    for( int i = 0; i < num_proc; i++ )
    {
        ierr = MPI_Irecv( &sizes_recv[i], 1, MPI_INT, procs[i], tag, cm, &recv_req[i] );
        if( ierr ) return MB_FILE_WRITE_ERROR;
    }
 
    // Send sizes
    assert( num_proc == (int)send_data.size() );
 
    result.resize( num_proc );
    for( int i = 0; i < num_proc; i++ )
    {
        sizes_send[i] = send_data[i].size();
        ierr          = MPI_Isend( &sizes_send[i], 1, MPI_INT, buffProcs[i], tag, cm, &send_req[i] );
        if( ierr ) return MB_FILE_WRITE_ERROR;
    }
 
    // Receive sizes
    std::vector< MPI_Status > stat( num_proc );
    ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
    if( ierr ) return MB_FILE_WRITE_ERROR;
 
    // Wait until all sizes are sent (clean up pending req's)
    ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
    if( ierr ) return MB_FILE_WRITE_ERROR;
 
    // Set up to receive data
    for( int i = 0; i < num_proc; i++ )
    {
        result[i].resize( sizes_recv[i] );
        ierr = MPI_Irecv( (void*)( &( result[i][0] ) ), sizeof( SharedEntityData ) * sizes_recv[i], MPI_UNSIGNED_CHAR,
                          buffProcs[i], tag, cm, &recv_req[i] );
        if( ierr ) return MB_FILE_WRITE_ERROR;
    }
 
    // Send data
    for( int i = 0; i < num_proc; i++ )
    {
        ierr = MPI_Isend( (void*)( &( send_data[i][0] ) ), sizeof( SharedEntityData ) * sizes_send[i],
                          MPI_UNSIGNED_CHAR, buffProcs[i], tag, cm, &send_req[i] );
        if( ierr ) return MB_FILE_WRITE_ERROR;
    }
 
    // Receive data
    ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
    if( ierr ) return MB_FILE_WRITE_ERROR;
 
    // Wait until everything is sent to release send buffers
    ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
    if( ierr ) return MB_FILE_WRITE_ERROR;
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::check_all_shared_handles( bool print_em )
{
    // Get all shared ent data from other procs
    std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() );
 
    ErrorCode result;
    bool done = false;
 
    while( !done )
    {
        result = check_local_shared();
        if( MB_SUCCESS != result )
        {
            done = true;
            continue;
        }
 
        result = pack_shared_handles( send_data );
        if( MB_SUCCESS != result )
        {
            done = true;
            continue;
        }
 
        result = exchange_all_shared_handles( send_data, shents );
        if( MB_SUCCESS != result )
        {
            done = true;
            continue;
        }
 
        if( !shents.empty() ) result = check_my_shared_handles( shents );
        done = true;
    }
 
    if( MB_SUCCESS != result && print_em )
    {
#ifdef MOAB_HAVE_HDF5
        std::ostringstream ent_str;
        ent_str << "mesh." << procConfig.proc_rank() << ".h5m";
        mbImpl->write_mesh( ent_str.str().c_str() );
#endif
    }
 
    return result;
}
 
ErrorCode ParallelComm::check_local_shared()
{
    // Do some checks on shared entities to make sure things look
    // consistent
 
    // Check that non-vertex shared entities are shared by same procs as all
    // their vertices
    // std::pair<Range::const_iterator,Range::const_iterator> vert_it =
    //    sharedEnts.equal_range(MBVERTEX);
    std::vector< EntityHandle > dum_connect;
    const EntityHandle* connect;
    int num_connect;
    int tmp_procs[MAX_SHARING_PROCS];
    EntityHandle tmp_hs[MAX_SHARING_PROCS];
    std::set< int > tmp_set, vset;
    int num_ps;
    ErrorCode result;
    unsigned char pstat;
    std::vector< EntityHandle > bad_ents;
    std::vector< std::string > errors;
 
    std::set< EntityHandle >::iterator vit;
    for( vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
    {
        // Get sharing procs for this ent
        result = get_sharing_data( *vit, tmp_procs, tmp_hs, pstat, num_ps );
        if( MB_SUCCESS != result )
        {
            bad_ents.push_back( *vit );
            errors.push_back( std::string( "Failure getting sharing data." ) );
            continue;
        }
 
        bool bad = false;
        // Entity must be shared
        if( !( pstat & PSTATUS_SHARED ) )
            errors.push_back( std::string( "Entity should be shared but isn't." ) ), bad = true;
 
        // If entity is not owned this must not be first proc
        if( pstat & PSTATUS_NOT_OWNED && tmp_procs[0] == (int)procConfig.proc_rank() )
            errors.push_back( std::string( "Entity not owned but is first proc." ) ), bad = true;
 
        // If entity is owned and multishared, this must be first proc
        if( !( pstat & PSTATUS_NOT_OWNED ) && pstat & PSTATUS_MULTISHARED &&
            ( tmp_procs[0] != (int)procConfig.proc_rank() || tmp_hs[0] != *vit ) )
            errors.push_back( std::string( "Entity owned and multishared but not first proc or not first handle." ) ),
                bad = true;
 
        if( bad )
        {
            bad_ents.push_back( *vit );
            continue;
        }
 
        EntityType type = mbImpl->type_from_handle( *vit );
        if( type == MBVERTEX || type == MBENTITYSET ) continue;
 
        // Copy element's procs to vset and save size
        int orig_ps = num_ps;
        vset.clear();
        std::copy( tmp_procs, tmp_procs + num_ps, std::inserter( vset, vset.begin() ) );
 
        // Get vertices for this ent and intersection of sharing procs
        result = mbImpl->get_connectivity( *vit, connect, num_connect, false, &dum_connect );
        if( MB_SUCCESS != result )
        {
            bad_ents.push_back( *vit );
            errors.push_back( std::string( "Failed to get connectivity." ) );
            continue;
        }
 
        for( int i = 0; i < num_connect; i++ )
        {
            result = get_sharing_data( connect[i], tmp_procs, NULL, pstat, num_ps );
            if( MB_SUCCESS != result )
            {
                bad_ents.push_back( *vit );
                continue;
            }
            if( !num_ps )
            {
                vset.clear();
                break;
            }
            std::sort( tmp_procs, tmp_procs + num_ps );
            tmp_set.clear();
            std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(),
                                   std::inserter( tmp_set, tmp_set.end() ) );
            vset.swap( tmp_set );
            if( vset.empty() ) break;
        }
 
        // Intersect them; should be the same size as orig_ps
        tmp_set.clear();
        std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(),
                               std::inserter( tmp_set, tmp_set.end() ) );
        if( orig_ps != (int)tmp_set.size() )
        {
            errors.push_back( std::string( "Vertex proc set not same size as entity proc set." ) );
            bad_ents.push_back( *vit );
            for( int i = 0; i < num_connect; i++ )
            {
                bad_ents.push_back( connect[i] );
                errors.push_back( std::string( "vertex in connect" ) );
            }
        }
    }
 
    if( !bad_ents.empty() )
    {
        std::cout << "Found bad entities in check_local_shared, proc rank " << procConfig.proc_rank() << ","
                  << std::endl;
        std::vector< std::string >::iterator sit;
        std::vector< EntityHandle >::iterator rit;
        for( rit = bad_ents.begin(), sit = errors.begin(); rit != bad_ents.end(); ++rit, ++sit )
        {
            list_entities( &( *rit ), 1 );
            std::cout << "Reason: " << *sit << std::endl;
        }
        return MB_FAILURE;
    }
 
    // To do: check interface sets
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::check_all_shared_handles( ParallelComm** pcs, int num_pcs )
{
    std::vector< std::vector< std::vector< SharedEntityData > > > shents, send_data;
    ErrorCode result = MB_SUCCESS, tmp_result;
 
    // Get all shared ent data from each proc to all other procs
    send_data.resize( num_pcs );
    for( int p = 0; p < num_pcs; p++ )
    {
        tmp_result = pcs[p]->pack_shared_handles( send_data[p] );
        if( MB_SUCCESS != tmp_result ) result = tmp_result;
    }
    if( MB_SUCCESS != result ) return result;
 
    // Move the data sorted by sending proc to data sorted by receiving proc
    shents.resize( num_pcs );
    for( int p = 0; p < num_pcs; p++ )
        shents[p].resize( pcs[p]->buffProcs.size() );
 
    for( int p = 0; p < num_pcs; p++ )
    {
        for( unsigned int idx_p = 0; idx_p < pcs[p]->buffProcs.size(); idx_p++ )
        {
            // Move send_data[p][to_p] to shents[to_p][idx_p]
            int to_p      = pcs[p]->buffProcs[idx_p];
            int top_idx_p = pcs[to_p]->get_buffers( p );
            assert( -1 != top_idx_p );
            shents[to_p][top_idx_p] = send_data[p][idx_p];
        }
    }
 
    for( int p = 0; p < num_pcs; p++ )
    {
        std::ostringstream ostr;
        ostr << "Processor " << p << " bad entities:";
        tmp_result = pcs[p]->check_my_shared_handles( shents[p], ostr.str().c_str() );
        if( MB_SUCCESS != tmp_result ) result = tmp_result;
    }
 
    return result;
}
 
ErrorCode ParallelComm::check_my_shared_handles( std::vector< std::vector< SharedEntityData > >& shents,
                                                 const char* prefix )
{
    // Now check against what I think data should be
    // Get all shared entities
    ErrorCode result;
    Range all_shared;
    std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( all_shared ) );
    std::vector< EntityHandle > dum_vec;
    all_shared.erase( all_shared.upper_bound( MBPOLYHEDRON ), all_shared.end() );
 
    Range bad_ents, local_shared;
    std::vector< SharedEntityData >::iterator vit;
    unsigned char tmp_pstat;
    for( unsigned int i = 0; i < shents.size(); i++ )
    {
        int other_proc = buffProcs[i];
        result         = get_shared_entities( other_proc, local_shared );
        if( MB_SUCCESS != result ) return result;
        for( vit = shents[i].begin(); vit != shents[i].end(); ++vit )
        {
            EntityHandle localh = vit->local, remoteh = vit->remote, dumh;
            local_shared.erase( localh );
            result = get_remote_handles( true, &localh, &dumh, 1, other_proc, dum_vec );
            if( MB_SUCCESS != result || dumh != remoteh ) bad_ents.insert( localh );
            result = get_pstatus( localh, tmp_pstat );
            if( MB_SUCCESS != result || ( !( tmp_pstat & PSTATUS_NOT_OWNED ) && (unsigned)vit->owner != rank() ) ||
                ( tmp_pstat & PSTATUS_NOT_OWNED && (unsigned)vit->owner == rank() ) )
                bad_ents.insert( localh );
        }
 
        if( !local_shared.empty() ) bad_ents.merge( local_shared );
    }
 
    if( !bad_ents.empty() )
    {
        if( prefix ) std::cout << prefix << std::endl;
        list_entities( bad_ents );
        return MB_FAILURE;
    }
    else
        return MB_SUCCESS;
}
 
ErrorCode ParallelComm::get_shared_entities( int other_proc,
                                             Range& shared_ents,
                                             int dim,
                                             const bool iface,
                                             const bool owned_filter )
{
    shared_ents.clear();
    ErrorCode result = MB_SUCCESS;
 
    // Dimension
    if( -1 != dim )
    {
        DimensionPair dp = CN::TypeDimensionMap[dim];
        Range dum_range;
        std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( dum_range ) );
        shared_ents.merge( dum_range.lower_bound( dp.first ), dum_range.upper_bound( dp.second ) );
    }
    else
        std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) );
 
    // Filter by iface
    if( iface )
    {
        result = filter_pstatus( shared_ents, PSTATUS_INTERFACE, PSTATUS_AND );MB_CHK_SET_ERR( result, "Failed to filter by iface" );
    }
 
    // Filter by owned
    if( owned_filter )
    {
        result = filter_pstatus( shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed to filter by owned" );
    }
 
    // Filter by proc
    if( -1 != other_proc )
    {
        result = filter_pstatus( shared_ents, PSTATUS_SHARED, PSTATUS_AND, other_proc );MB_CHK_SET_ERR( result, "Failed to filter by proc" );
    }
 
    return result;
}
 
ErrorCode ParallelComm::clean_shared_tags( std::vector< Range* >& exchange_ents )
{
    for( unsigned int i = 0; i < exchange_ents.size(); i++ )
    {
        Range* ents        = exchange_ents[i];
        int num_ents       = ents->size();
        Range::iterator it = ents->begin();
 
        for( int n = 0; n < num_ents; n++ )
        {
            int sharing_proc;
            ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &( *ents->begin() ), 1, &sharing_proc );
            if( result != MB_TAG_NOT_FOUND && sharing_proc == -1 )
            {
                result = mbImpl->tag_delete_data( sharedp_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedp tag data" );
                result = mbImpl->tag_delete_data( sharedh_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedh tag data" );
                result = mbImpl->tag_delete_data( pstatus_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete pstatus tag data" );
            }
            ++it;
        }
    }
 
    return MB_SUCCESS;
}
 
void ParallelComm::set_debug_verbosity( int verb )
{
    myDebug->set_verbosity( verb );
}
 
int ParallelComm::get_debug_verbosity()
{
    return myDebug->get_verbosity();
}
 
ErrorCode ParallelComm::get_entityset_procs( EntityHandle set, std::vector< unsigned >& ranks ) const
{
    return sharedSetData->get_sharing_procs( set, ranks );
}
 
ErrorCode ParallelComm::get_entityset_owner( EntityHandle entity_set,
                                             unsigned& owner_rank,
                                             EntityHandle* remote_handle ) const
{
    if( remote_handle )
        return sharedSetData->get_owner( entity_set, owner_rank, *remote_handle );
    else
        return sharedSetData->get_owner( entity_set, owner_rank );
}
 
ErrorCode ParallelComm::get_entityset_local_handle( unsigned owning_rank,
                                                    EntityHandle remote_handle,
                                                    EntityHandle& local_handle ) const
{
    return sharedSetData->get_local_handle( owning_rank, remote_handle, local_handle );
}
 
ErrorCode ParallelComm::get_shared_sets( Range& result ) const
{
    return sharedSetData->get_shared_sets( result );
}
 
ErrorCode ParallelComm::get_entityset_owners( std::vector< unsigned >& ranks ) const
{
    return sharedSetData->get_owning_procs( ranks );
}
 
ErrorCode ParallelComm::get_owned_sets( unsigned owning_rank, Range& sets_out ) const
{
    return sharedSetData->get_shared_sets( owning_rank, sets_out );
}
 
ErrorCode ParallelComm::gather_data( Range& gather_ents,
                                     Tag& tag_handle,
                                     Tag id_tag,
                                     EntityHandle gather_set,
                                     int root_proc_rank )
{
    int dim           = mbImpl->dimension_from_handle( *gather_ents.begin() );
    int bytes_per_tag = 0;
    ErrorCode rval    = mbImpl->tag_get_bytes( tag_handle, bytes_per_tag );
    if( rval != MB_SUCCESS ) return rval;
 
    int sz_buffer         = sizeof( int ) + gather_ents.size() * ( sizeof( int ) + bytes_per_tag );
    void* senddata        = malloc( sz_buffer );
    ( (int*)senddata )[0] = (int)gather_ents.size();
    int* ptr_int          = (int*)senddata + 1;
    rval                  = mbImpl->tag_get_data( id_tag, gather_ents, (void*)ptr_int );
    if( rval != MB_SUCCESS ) return rval;
    ptr_int = (int*)( senddata ) + 1 + gather_ents.size();
    rval    = mbImpl->tag_get_data( tag_handle, gather_ents, (void*)ptr_int );
    if( rval != MB_SUCCESS ) return rval;
    std::vector< int > displs( proc_config().proc_size(), 0 );
    MPI_Gather( &sz_buffer, 1, MPI_INT, &displs[0], 1, MPI_INT, root_proc_rank, comm() );
    std::vector< int > recvcnts( proc_config().proc_size(), 0 );
    std::copy( displs.begin(), displs.end(), recvcnts.begin() );
    std::partial_sum( displs.begin(), displs.end(), displs.begin() );
    std::vector< int >::iterator lastM1 = displs.end() - 1;
    std::copy_backward( displs.begin(), lastM1, displs.end() );
    // std::copy_backward(displs.begin(), --displs.end(), displs.end());
    displs[0] = 0;
 
    if( (int)rank() != root_proc_rank )
        MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, NULL, NULL, NULL, MPI_BYTE, root_proc_rank, comm() );
    else
    {
        Range gents;
        mbImpl->get_entities_by_dimension( gather_set, dim, gents );
        int recvbuffsz = gents.size() * ( bytes_per_tag + sizeof( int ) ) + proc_config().proc_size() * sizeof( int );
        void* recvbuf  = malloc( recvbuffsz );
        MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, recvbuf, &recvcnts[0], &displs[0], MPI_BYTE, root_proc_rank,
                     comm() );
 
        void* gvals = NULL;
 
        // Test whether gents has multiple sequences
        bool multiple_sequences = false;
        if( gents.psize() > 1 )
            multiple_sequences = true;
        else
        {
            int count;
            rval = mbImpl->tag_iterate( tag_handle, gents.begin(), gents.end(), count, gvals );
            assert( NULL != gvals );
            assert( count > 0 );
            if( (size_t)count != gents.size() )
            {
                multiple_sequences = true;
                gvals              = NULL;
            }
        }
 
        // If gents has multiple sequences, create a temp buffer for gathered values
        if( multiple_sequences )
        {
            gvals = malloc( gents.size() * bytes_per_tag );
            assert( NULL != gvals );
        }
 
        for( int i = 0; i != (int)size(); i++ )
        {
            int numents   = *(int*)( ( (char*)recvbuf ) + displs[i] );
            int* id_ptr   = (int*)( ( (char*)recvbuf ) + displs[i] + sizeof( int ) );
            char* val_ptr = (char*)( id_ptr + numents );
            for( int j = 0; j != numents; j++ )
            {
                int idx = id_ptr[j];
                memcpy( (char*)gvals + ( idx - 1 ) * bytes_per_tag, val_ptr + j * bytes_per_tag, bytes_per_tag );
            }
        }
 
        // Free the receive buffer
        free( recvbuf );
 
        // If gents has multiple sequences, copy tag data (stored in the temp buffer) to each
        // sequence separately
        if( multiple_sequences )
        {
            Range::iterator iter = gents.begin();
            size_t start_idx     = 0;
            while( iter != gents.end() )
            {
                int count;
                void* ptr;
                rval = mbImpl->tag_iterate( tag_handle, iter, gents.end(), count, ptr );
                assert( NULL != ptr );
                assert( count > 0 );
                memcpy( (char*)ptr, (char*)gvals + start_idx * bytes_per_tag, bytes_per_tag * count );
 
                iter += count;
                start_idx += count;
            }
            assert( start_idx == gents.size() );
 
            // Free the temp buffer
            free( gvals );
        }
    }
 
    // Free the send data
    free( senddata );
 
    return MB_SUCCESS;
}
 
/*
 * This call is collective, so we will use the message ids for tag communications;
 * they are similar, but simpler
 * Pack the number of edges, the remote edge handles, then for each edge, the number
 *    of intersection points, and then 3 doubles for each intersection point
 * On average, there is one intx point per edge, in some cases 2, in some cases 0
 *   so on average, the message size is num_edges * (sizeof(eh) + sizeof(int) + 1*3*sizeof(double))
 *          = num_edges * (8 + 4 + 24)
 */
ErrorCode ParallelComm::settle_intersection_points( Range& edges,
                                                    Range& shared_edges_owned,
                                                    std::vector< std::vector< EntityHandle >* >& extraNodesVec,
                                                    double tolerance )
{
    // The index of an edge in the edges Range will give the index for extraNodesVec
    // the strategy of this follows exchange tags strategy:
    ErrorCode result;
    int success;
 
    myDebug->tprintf( 1, "Entering settle_intersection_points\n" );
 
    // Get all procs interfacing to this proc
    std::set< unsigned int > exch_procs;
    result = get_comm_procs( exch_procs );
 
    // Post ghost irecv's for all interface procs
    // Index requests the same as buffer/sharing procs indices
    std::vector< MPI_Request > recv_intx_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
    std::vector< unsigned int >::iterator sit;
    int ind;
 
    reset_all_buffers();
    int incoming = 0;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        incoming++;
        PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
                           MB_MESG_TAGS_SIZE, incoming );
 
        success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
                             MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_intx_reqs[3 * ind] );
        if( success != MPI_SUCCESS )
        {
            MB_SET_ERR( MB_FAILURE, "Failed to post irecv in settle intersection point" );
        }
    }
 
    // Pack and send intersection points from this proc to others
    // Make sendReqs vector to simplify initialization
    sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
 
    // Take all shared entities if incoming list is empty
    Range& entities = shared_edges_owned;
 
    int dum_ack_buff;
 
    for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
    {
        Range edges_to_send = entities;
 
        // Get ents shared by proc *sit
        result = filter_pstatus( edges_to_send, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
 
        // Remote nonowned entities; not needed, edges are already owned by this proc
 
        // Pack the data
        // Reserve space on front for size and for initial buff size
        Buffer* buff = localOwnedBuffs[ind];
        buff->reset_ptr( sizeof( int ) );
 
        /*result = pack_intx_points(edges_to_send, edges, extraNodesVec,
            localOwnedBuffs[ind], *sit);*/
 
        // Count first data, and see if it is enough room?
        // Send the remote handles
        std::vector< EntityHandle > dum_remote_edges( edges_to_send.size() );
        /*
         *  get_remote_handles(const bool store_remote_handles,
                                   EntityHandle *from_vec,
                                   EntityHandle *to_vec_tmp,
                                   int num_ents, int to_proc,
                                   const std::vector<EntityHandle> &new_ents);
         */
        // We are sending count, num edges, remote edges handles, and then, for each edge:
        //          -- nb intx points, 3*nbintPointsforEdge "doubles"
        std::vector< EntityHandle > dum_vec;
        result = get_remote_handles( true, edges_to_send, &dum_remote_edges[0], *sit, dum_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles" );
        int count = 4;  // Size of data
        count += sizeof( int ) * (int)edges_to_send.size();
        count += sizeof( EntityHandle ) * (int)edges_to_send.size();  // We will send the remote handles
        for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit )
        {
            EntityHandle edge                       = *eit;
            unsigned int indx                       = edges.find( edge ) - edges.begin();
            std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
            count += (int)intx_nodes.size() * 3 * sizeof( double );  // 3 integer for each entity handle
        }
        //
        buff->check_space( count );
        PACK_INT( buff->buff_ptr, edges_to_send.size() );
        PACK_EH( buff->buff_ptr, &dum_remote_edges[0], dum_remote_edges.size() );
        for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit )
        {
            EntityHandle edge = *eit;
            // Pack the remote edge
            unsigned int indx                       = edges.find( edge ) - edges.begin();
            std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
            PACK_INT( buff->buff_ptr, intx_nodes.size() );
 
            result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), (double*)buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get coords" );
            buff->buff_ptr += 3 * sizeof( double ) * intx_nodes.size();
        }
 
        // Done packing the intx points and remote edges
        buff->set_stored_size();
 
        // Now send it
        result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
                              recv_intx_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
    }
 
    // Receive/unpack intx points
    while( incoming )
    {
        MPI_Status status;
        int index_in_recv_requests;
        PRINT_DEBUG_WAITANY( recv_intx_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
        success = MPI_Waitany( 3 * buffProcs.size(), &recv_intx_reqs[0], &index_in_recv_requests, &status );
        if( MPI_SUCCESS != success )
        {
            MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
        }
        // Processor index in the list is divided by 3
        ind = index_in_recv_requests / 3;
 
        PRINT_DEBUG_RECD( status );
 
        // OK, received something; decrement incoming counter
        incoming--;
 
        bool done = false;
        result    = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
                                 recv_intx_reqs[3 * ind + 1],  // This is for receiving the second message
                                 recv_intx_reqs[3 * ind + 2],  // This would be for ack, but it is not
                                                               // used; consider removing it
                                 incoming, localOwnedBuffs[ind],
                                 sendReqs[3 * ind + 1],  // Send request for sending the second message
                                 sendReqs[3 * ind + 2],  // This is for sending the ack
                                 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
        if( done )
        {
            Buffer* buff = remoteOwnedBuffs[ind];
            buff->reset_ptr( sizeof( int ) );
            /*result = unpack_tags(remoteOwnedBuffs[ind/2]->buff_ptr, dum_vec, true,
                buffProcs[ind/2]);*/
            // Unpack now the edges and vertex info; compare with the existing vertex positions
 
            int num_edges;
 
            UNPACK_INT( buff->buff_ptr, num_edges );
            std::vector< EntityHandle > rec_edges;
            rec_edges.resize( num_edges );
            UNPACK_EH( buff->buff_ptr, &rec_edges[0], num_edges );
            for( int i = 0; i < num_edges; i++ )
            {
                EntityHandle edge                       = rec_edges[i];
                unsigned int indx                       = edges.find( edge ) - edges.begin();
                std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
                // Now get the number of nodes on this (now local) edge
                int nverts;
                UNPACK_INT( buff->buff_ptr, nverts );
                std::vector< double > pos_from_owner;
                pos_from_owner.resize( 3 * nverts );
                UNPACK_DBLS( buff->buff_ptr, &pos_from_owner[0], 3 * nverts );
                std::vector< double > current_positions( 3 * intx_nodes.size() );
                result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), &current_positions[0] );MB_CHK_SET_ERR( result, "Failed to get current positions" );
                // Now, look at what we have in current pos, compare to pos from owner, and reset
                for( int k = 0; k < (int)intx_nodes.size(); k++ )
                {
                    double* pk = &current_positions[3 * k];
                    // Take the current pos k, and settle among the ones from owner:
                    bool found = false;
                    for( int j = 0; j < nverts && !found; j++ )
                    {
                        double* pj   = &pos_from_owner[3 * j];
                        double dist2 = ( pk[0] - pj[0] ) * ( pk[0] - pj[0] ) + ( pk[1] - pj[1] ) * ( pk[1] - pj[1] ) +
                                       ( pk[2] - pj[2] ) * ( pk[2] - pj[2] );
                        if( dist2 < tolerance )
                        {
                            pk[0] = pj[0];
                            pk[1] = pj[1];
                            pk[2] = pj[2];  // Correct it!
                            found = true;
                            break;
                        }
                    }
                    if( !found )
                    {
#ifndef NDEBUG
                        std::cout << " pk:" << pk[0] << " " << pk[1] << " " << pk[2] << " not found \n";
#endif
                        result = MB_FAILURE;
                    }
                }
                // After we are done resetting, we can set the new positions of nodes:
                result = mbImpl->set_coords( &intx_nodes[0], (int)intx_nodes.size(), &current_positions[0] );MB_CHK_SET_ERR( result, "Failed to set new current positions" );
            }
        }
    }
 
    // OK, now wait
    if( myDebug->get_verbosity() == 5 )
    {
        success = MPI_Barrier( procConfig.proc_comm() );
    }
    else
    {
        MPI_Status status[3 * MAX_SHARING_PROCS];
        success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
    }
    if( MPI_SUCCESS != success )
    {
        MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
    }
 
    myDebug->tprintf( 1, "Exiting settle_intersection_points" );
 
    return MB_SUCCESS;
}
 
ErrorCode ParallelComm::delete_entities( Range& to_delete )
{
    // Will not look at shared sets yet, but maybe we should
    // First, see if any of the entities to delete is shared; then inform the other processors
    // about their fate (to be deleted), using a crystal router transfer
    ErrorCode rval = MB_SUCCESS;
    unsigned char pstat;
    EntityHandle tmp_handles[MAX_SHARING_PROCS];
    int tmp_procs[MAX_SHARING_PROCS];
    unsigned int num_ps;
    TupleList ents_to_delete;
    ents_to_delete.initialize( 1, 0, 1, 0, to_delete.size() * ( MAX_SHARING_PROCS + 1 ) );  // A little bit of overkill
    ents_to_delete.enableWriteAccess();
    unsigned int i = 0;
    for( Range::iterator it = to_delete.begin(); it != to_delete.end(); ++it )
    {
        EntityHandle eh = *it;  // Entity to be deleted
 
        rval = get_sharing_data( eh, tmp_procs, tmp_handles, pstat, num_ps );
        if( rval != MB_SUCCESS || num_ps == 0 ) continue;
        // Add to the tuple list the information to be sent (to the remote procs)
        for( unsigned int p = 0; p < num_ps; p++ )
        {
            ents_to_delete.vi_wr[i]  = tmp_procs[p];
            ents_to_delete.vul_wr[i] = (unsigned long)tmp_handles[p];
            i++;
            ents_to_delete.inc_n();
        }
    }
 
    gs_data::crystal_data* cd = this->procConfig.crystal_router();
    // All communication happens here; no other mpi calls
    // Also, this is a collective call
    rval = cd->gs_transfer( 1, ents_to_delete, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" );
 
    // Add to the range of ents to delete the new ones that were sent from other procs
    unsigned int received = ents_to_delete.get_n();
    for( i = 0; i < received; i++ )
    {
        // int from = ents_to_delete.vi_rd[i];
        unsigned long valrec = ents_to_delete.vul_rd[i];
        to_delete.insert( (EntityHandle)valrec );
    }
    rval = mbImpl->delete_entities( to_delete );MB_CHK_SET_ERR( rval, "Error in deleting actual entities" );
 
    std::set< EntityHandle > good_ents;
    for( std::set< EntityHandle >::iterator sst = sharedEnts.begin(); sst != sharedEnts.end(); sst++ )
    {
        EntityHandle eh = *sst;
        int index       = to_delete.index( eh );
        if( -1 == index ) good_ents.insert( eh );
    }
    sharedEnts = good_ents;
 
    // What about shared sets? Who is updating them?
    return MB_SUCCESS;
}
 
void ParallelComm::print_pstatus( unsigned char pstat, std::string& ostr )
{
    std::ostringstream str;
    int num = 0;
#define ppstat( a, b )             \
    {                              \
        if( pstat & ( a ) )        \
        {                          \
            if( num ) str << ", "; \
            str << ( b );          \
            num++;                 \
        }                          \
    }
 
    ppstat( PSTATUS_NOT_OWNED, "NOT_OWNED" );
    ppstat( PSTATUS_SHARED, "SHARED" );
    ppstat( PSTATUS_MULTISHARED, "MULTISHARED" );
    ppstat( PSTATUS_INTERFACE, "INTERFACE" );
    ppstat( PSTATUS_GHOST, "GHOST" );
 
    ostr = str.str();
}
 
void ParallelComm::print_pstatus( unsigned char pstat )
{
    std::string str;
    print_pstatus( pstat, str );
    std::cout << str.c_str() << std::endl;
}
 
ErrorCode ParallelComm::correct_thin_ghost_layers()
{
 
    // Get all shared ent data from other procs
    std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() );
 
    // will work only on multi-shared tags  sharedps_tag(), sharedhs_tag();
 
    /*
     *   domain0 | domain1 | domain2 | domain3
     *   vertices from domain 1 and 2 are visible from both 0 and 3, but
     *   domain 0 might not have info about multi-sharing from domain 3
     *   so we will force that domain 0 vertices owned by 1 and 2 have information
     *   about the domain 3 sharing
     *
     *   SharedEntityData will have :
     *    struct SharedEntityData {
            EntityHandle local;  // this is same meaning, for the proc we sent to, it is local
            EntityHandle remote; // this will be the far away handle that will need to be added
            EntityID owner;      // this will be the remote proc
          };
          // so we need to add data like this:
           a multishared entity owned by proc x will have data like
             multishared procs:  proc x, a, b, c
             multishared handles:     h1, h2, h3, h4
             we will need to send data from proc x like this:
               to proc a we will send
                 (h2, h3, b), (h2, h4, c)
               to proc b we will send
                  (h3, h2, a), (h3, h4, c)
               to proc c we will send
                  (h4, h2, a), (h4, h3, b)
     *
     */
 
    ErrorCode result = MB_SUCCESS;
    int ent_procs[MAX_SHARING_PROCS + 1];
    EntityHandle handles[MAX_SHARING_PROCS + 1];
    int num_sharing;
    SharedEntityData tmp;
 
    for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i )
    {
 
        unsigned char pstat;
        result = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" );
        if( !( pstat & PSTATUS_MULTISHARED ) ||
            num_sharing <= 2 )  // if not multishared, skip, it should have no problems
            continue;
        // we should skip the ones that are not owned locally
        // the owned ones will have the most multi-shared info, because the info comes from other
        // remote processors
        if( pstat & PSTATUS_NOT_OWNED ) continue;
        for( int j = 1; j < num_sharing; j++ )
        {
            // we will send to proc
            int send_to_proc = ent_procs[j];  //
            tmp.local        = handles[j];
            int ind          = get_buffers( send_to_proc );
            assert( -1 != ind );  // THIS SHOULD NEVER HAPPEN
            for( int k = 1; k < num_sharing; k++ )
            {
                // do not send to self proc
                if( j == k ) continue;
                tmp.remote = handles[k];  // this will be the handle of entity on proc
                tmp.owner  = ent_procs[k];
                send_data[ind].push_back( tmp );
            }
        }
    }
 
    result = exchange_all_shared_handles( send_data, shents );MB_CHK_ERR( result );
 
    // loop over all shents and add if vertex type, add if missing
    for( size_t i = 0; i < shents.size(); i++ )
    {
        std::vector< SharedEntityData >& shEnts = shents[i];
        for( size_t j = 0; j < shEnts.size(); j++ )
        {
            tmp = shEnts[j];
            // basically, check the shared data for tmp.local entity
            // it should have inside the tmp.owner and tmp.remote
            EntityHandle eh = tmp.local;
            unsigned char pstat;
            result = get_sharing_data( eh, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" );
            // see if the proc tmp.owner is in the list of ent_procs; if not, we have to increase
            // handles, and ent_procs; and set
 
            int proc_remote = tmp.owner;  //
            if( std::find( ent_procs, ent_procs + num_sharing, proc_remote ) == ent_procs + num_sharing )
            {
                // so we did not find on proc
#ifndef NDEBUG
                if( myDebug->get_verbosity() == 3 )
                    std::cout << "THIN GHOST: we did not find on proc " << rank() << " for shared ent " << eh
                              << " the proc " << proc_remote << "\n";
#endif
                // increase num_sharing, and set the multi-shared tags
                if( num_sharing >= MAX_SHARING_PROCS ) return MB_FAILURE;
                handles[num_sharing]       = tmp.remote;
                handles[num_sharing + 1]   = 0;  // end of list
                ent_procs[num_sharing]     = tmp.owner;
                ent_procs[num_sharing + 1] = -1;  // this should be already set
                result                     = mbImpl->tag_set_data( sharedps_tag(), &eh, 1, ent_procs );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" );
                result = mbImpl->tag_set_data( sharedhs_tag(), &eh, 1, handles );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" );
                if( 2 == num_sharing )  // it means the sharedp and sharedh tags were set with a
                                        // value non default
                {
                    // so entity eh was simple shared before, we need to set those dense tags back
                    // to default
                    //  values
                    EntityHandle zero = 0;
                    int no_proc       = -1;
                    result            = mbImpl->tag_set_data( sharedp_tag(), &eh, 1, &no_proc );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
                    result = mbImpl->tag_set_data( sharedh_tag(), &eh, 1, &zero );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
                    // also, add multishared pstatus tag
                    // also add multishared status to pstatus
                    pstat  = pstat | PSTATUS_MULTISHARED;
                    result = mbImpl->tag_set_data( pstatus_tag(), &eh, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
                }
            }
        }
    }
    return MB_SUCCESS;
}
}  // namespace moab