1 #include "moab/Interface.hpp"
2 #include "moab/ParallelComm.hpp"
3 #include "moab/WriteUtilIface.hpp"
4 #include "moab/ReadUtilIface.hpp"
5 #include "SequenceManager.hpp"
6 #include "moab/Error.hpp"
7 #include "EntitySequence.hpp"
8 #include "MBTagConventions.hpp"
9 #include "moab/Skinner.hpp"
10 #include "MBParallelConventions.h"
11 #include "moab/Core.hpp"
12 #include "ElementSequence.hpp"
13 #include "moab/CN.hpp"
14 #include "moab/RangeMap.hpp"
15 #include "moab/MeshTopoUtil.hpp"
16 #include "TagInfo.hpp"
17 #include "DebugOutput.hpp"
18 #include "SharedSetData.hpp"
19 #include "moab/ScdInterface.hpp"
20 #include "moab/TupleList.hpp"
21 #include "moab/gs.hpp"
22
23 #include <iostream>
24 #include <sstream>
25 #include <algorithm>
26 #include <functional>
27 #include <numeric>
28
29 #include <cmath>
30 #include <cstdlib>
31 #include <cassert>
32
33 #ifdef MOAB_HAVE_MPI
34 #include "moab_mpi.h"
35 #endif
36 #ifdef MOAB_HAVE_MPE
37 #include "mpe.h"
38 int IFACE_START, IFACE_END;
39 int GHOST_START, GHOST_END;
40 int SHAREDV_START, SHAREDV_END;
41 int RESOLVE_START, RESOLVE_END;
42 int ENTITIES_START, ENTITIES_END;
43 int RHANDLES_START, RHANDLES_END;
44 int OWNED_START, OWNED_END;
45 #endif
46
47 namespace moab
48 {
49
50 const unsigned int ParallelComm::INITIAL_BUFF_SIZE = 1024;
51
52 const int MAX_BCAST_SIZE = ( 1 << 28 );
53
54 std::vector< ParallelComm::Buffer* > msgs;
55 unsigned int __PACK_num = 0, __UNPACK_num = 0, __PACK_count = 0, __UNPACK_count = 0;
56 std::string __PACK_string, __UNPACK_string;
57
58 #ifdef DEBUG_PACKING_TIMES
59 #define PC( n, m ) \
60 { \
61 if( __PACK_num == (unsigned int)n && __PACK_string == m ) \
62 __PACK_count++; \
63 else \
64 { \
65 if( __PACK_count > 1 ) std::cerr << " (" << __PACK_count << "x)"; \
66 __PACK_count = 1; \
67 __PACK_string = m; \
68 __PACK_num = n; \
69 std::cerr << std::endl << "PACK: " << n << m; \
70 } \
71 }
72 #define UPC( n, m ) \
73 { \
74 if( __UNPACK_num == (unsigned int)n && __UNPACK_string == m ) \
75 __UNPACK_count++; \
76 else \
77 { \
78 if( __UNPACK_count > 1 ) std::cerr << "(" << __UNPACK_count << "x)"; \
79 __UNPACK_count = 1; \
80 __UNPACK_string = m; \
81 __UNPACK_num = n; \
82 std::cerr << std::endl << "UNPACK: " << n << m; \
83 } \
84 }
85 #else
86 #define PC( n, m )
87 #define UPC( n, m )
88 #endif
89
90 template < typename T >
91 static inline void UNPACK( unsigned char*& buff, T* val, size_t count )
92 {
93 memcpy( val, buff, count * sizeof( T ) );
94 buff += count * sizeof( T );
95 }
96
97 template < typename T >
98 static inline void PACK( unsigned char*& buff, const T* val, size_t count )
99 {
100 memcpy( buff, val, count * sizeof( T ) );
101 buff += count * sizeof( T );
102 }
103
104 static inline void PACK_INTS( unsigned char*& buff, const int* int_val, size_t num )
105 {
106 PACK( buff, int_val, num );
107 PC( num, " ints" );
108 }
109
110 static inline void PACK_INT( unsigned char*& buff, int int_val )
111 {
112 PACK_INTS( buff, &int_val, 1 );
113 }
114
115 static inline void PACK_DBLS( unsigned char*& buff, const double* dbl_val, size_t num )
116 {
117 PACK( buff, dbl_val, num );
118 PC( num, " doubles" );
119 }
120
121
122
123
124
125 static inline void PACK_EH( unsigned char*& buff, const EntityHandle* eh_val, size_t num )
126 {
127 PACK( buff, eh_val, num );
128 PC( num, " handles" );
129 }
130
131
132
133
134
135
136
137
138
139 static inline void PACK_VOID( unsigned char*& buff, const void* val, size_t num )
140 {
141 PACK( buff, reinterpret_cast< const unsigned char* >( val ), num );
142 PC( num, " void" );
143 }
144
145 static inline void PACK_BYTES( unsigned char*& buff, const void* val, int num )
146 {
147 PACK_INT( buff, num );
148 PACK_VOID( buff, val, num );
149 }
150
151 static inline void PACK_RANGE( unsigned char*& buff, const Range& rng )
152 {
153 PACK_INT( buff, rng.psize() );
154 Range::const_pair_iterator cit;
155 for( cit = rng.const_pair_begin(); cit != rng.const_pair_end(); ++cit )
156 {
157 EntityHandle eh[2] = { cit->first, cit->second };
158 PACK_EH( buff, eh, 2 );
159 }
160 PC( rng.psize(), "-subranged range" );
161 }
162
163 static inline void UNPACK_INTS( unsigned char*& buff, int* int_val, size_t num )
164 {
165 UNPACK( buff, int_val, num );
166 UPC( num, " ints" );
167 }
168
169 static inline void UNPACK_INT( unsigned char*& buff, int& int_val )
170 {
171 UNPACK_INTS( buff, &int_val, 1 );
172 }
173
174 static inline void UNPACK_DBLS( unsigned char*& buff, double* dbl_val, size_t num )
175 {
176 UNPACK( buff, dbl_val, num );
177 UPC( num, " doubles" );
178 }
179
180 static inline void UNPACK_DBL( unsigned char*& buff, double& dbl_val )
181 {
182 UNPACK_DBLS( buff, &dbl_val, 1 );
183 }
184
185 static inline void UNPACK_EH( unsigned char*& buff, EntityHandle* eh_val, size_t num )
186 {
187 UNPACK( buff, eh_val, num );
188 UPC( num, " handles" );
189 }
190
191
192
193
194
195
196
197
198
199 static inline void UNPACK_VOID( unsigned char*& buff, void* val, size_t num )
200 {
201 UNPACK( buff, reinterpret_cast< unsigned char* >( val ), num );
202 UPC( num, " void" );
203 }
204
205 static inline void UNPACK_TYPE( unsigned char*& buff, EntityType& type )
206 {
207 int int_type = MBMAXTYPE;
208 UNPACK_INT( buff, int_type );
209 type = static_cast< EntityType >( int_type );
210 assert( type >= MBVERTEX && type <= MBMAXTYPE );
211 }
212
213 static inline void UNPACK_RANGE( unsigned char*& buff, Range& rng )
214 {
215 int num_subs;
216 EntityHandle eh[2];
217 UNPACK_INT( buff, num_subs );
218 for( int i = 0; i < num_subs; i++ )
219 {
220 UPC( num_subs, "-subranged range" );
221 UNPACK_EH( buff, eh, 2 );
222 rng.insert( eh[0], eh[1] );
223 }
224 }
225
226 enum MBMessageTag
227 {
228 MB_MESG_ANY = MPI_ANY_TAG,
229 MB_MESG_ENTS_ACK,
230 MB_MESG_ENTS_SIZE,
231 MB_MESG_ENTS_LARGE,
232 MB_MESG_REMOTEH_ACK,
233 MB_MESG_REMOTEH_SIZE,
234 MB_MESG_REMOTEH_LARGE,
235 MB_MESG_TAGS_ACK,
236 MB_MESG_TAGS_SIZE,
237 MB_MESG_TAGS_LARGE
238 };
239
240 static inline size_t RANGE_SIZE( const Range& rng )
241 {
242 return 2 * sizeof( EntityHandle ) * rng.psize() + sizeof( int );
243 }
244
245 #define PRINT_DEBUG_ISEND( A, B, C, D, E ) print_debug_isend( ( A ), ( B ), ( C ), ( D ), ( E ) )
246 #define PRINT_DEBUG_IRECV( A, B, C, D, E, F ) print_debug_irecv( ( A ), ( B ), ( C ), ( D ), ( E ), ( F ) )
247 #define PRINT_DEBUG_RECD( A ) print_debug_recd( ( A ) )
248 #define PRINT_DEBUG_WAITANY( A, B, C ) print_debug_waitany( ( A ), ( B ), ( C ) )
249
250 void ParallelComm::print_debug_isend( int from, int to, unsigned char* buff, int tag, int sz )
251 {
252 myDebug->tprintf( 3, "Isend, %d->%d, buffer ptr = %p, tag=%d, size=%d\n", from, to, (void*)buff, tag, sz );
253 }
254
255 void ParallelComm::print_debug_irecv( int to, int from, unsigned char* buff, int sz, int tag, int incoming )
256 {
257 myDebug->tprintf( 3, "Irecv, %d<-%d, buffer ptr = %p, tag=%d, size=%d", to, from, (void*)buff, tag, sz );
258 if( tag < MB_MESG_REMOTEH_ACK )
259 myDebug->printf( 3, ", incoming1=%d\n", incoming );
260 else if( tag < MB_MESG_TAGS_ACK )
261 myDebug->printf( 3, ", incoming2=%d\n", incoming );
262 else
263 myDebug->printf( 3, ", incoming=%d\n", incoming );
264 }
265
266 void ParallelComm::print_debug_recd( MPI_Status status )
267 {
268 if( myDebug->get_verbosity() == 3 )
269 {
270 int this_count;
271 int success = MPI_Get_count( &status, MPI_UNSIGNED_CHAR, &this_count );
272 if( MPI_SUCCESS != success ) this_count = -1;
273 myDebug->tprintf( 3, "Received from %d, count = %d, tag = %d\n", status.MPI_SOURCE, this_count,
274 status.MPI_TAG );
275 }
276 }
277
278 void ParallelComm::print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc )
279 {
280 if( myDebug->get_verbosity() == 3 )
281 {
282 myDebug->tprintf( 3, "Waitany, p=%d, ", proc );
283 if( tag < MB_MESG_REMOTEH_ACK )
284 myDebug->print( 3, ", recv_ent_reqs=" );
285 else if( tag < MB_MESG_TAGS_ACK )
286 myDebug->print( 3, ", recv_remoteh_reqs=" );
287 else
288 myDebug->print( 3, ", recv_tag_reqs=" );
289 for( unsigned int i = 0; i < reqs.size(); i++ )
290 myDebug->printf( 3, " %p", (void*)(intptr_t)reqs[i] );
291 myDebug->print( 3, "\n" );
292 }
293 }
294
295
296 const char* PARTITIONING_PCOMM_TAG_NAME = "__PRTN_PCOMM";
297
298
311 #define PARALLEL_COMM_TAG_NAME "__PARALLEL_COMM"
312
313 ParallelComm::ParallelComm( Interface* impl, MPI_Comm cm, int* id )
314 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
315 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
316 myDebug( NULL )
317 {
318 initialize();
319 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
320 if( id ) *id = pcommID;
321 }
322
323 ParallelComm::ParallelComm( Interface* impl, std::vector< unsigned char >& , MPI_Comm cm, int* id )
324 : mbImpl( impl ), procConfig( cm ), sharedpTag( 0 ), sharedpsTag( 0 ), sharedhTag( 0 ), sharedhsTag( 0 ),
325 pstatusTag( 0 ), ifaceSetsTag( 0 ), partitionTag( 0 ), globalPartCount( -1 ), partitioningSet( 0 ),
326 myDebug( NULL )
327 {
328 initialize();
329 sharedSetData = new SharedSetData( *impl, pcommID, procConfig.proc_rank() );
330 if( id ) *id = pcommID;
331 }
332
333 ParallelComm::~ParallelComm()
334 {
335 remove_pcomm( this );
336 delete_all_buffers();
337 delete myDebug;
338 delete sharedSetData;
339 }
340
341 void ParallelComm::initialize()
342 {
343 Core* core = dynamic_cast< Core* >( mbImpl );
344 sequenceManager = core->sequence_manager();
345 mbImpl->query_interface( errorHandler );
346
347
348 int flag = 1;
349 int retval = MPI_Initialized( &flag );
350 if( MPI_SUCCESS != retval || !flag )
351 {
352 int argc = 0;
353 char** argv = NULL;
354
355
356 retval = MPI_Init( &argc, &argv );
357 assert( MPI_SUCCESS == retval );
358 }
359
360
361 buffProcs.reserve( MAX_SHARING_PROCS );
362 localOwnedBuffs.reserve( MAX_SHARING_PROCS );
363 remoteOwnedBuffs.reserve( MAX_SHARING_PROCS );
364
365 pcommID = add_pcomm( this );
366
367 if( !myDebug )
368 {
369 myDebug = new DebugOutput( "ParallelComm", std::cerr );
370 myDebug->set_rank( procConfig.proc_rank() );
371 }
372 }
373
374 int ParallelComm::add_pcomm( ParallelComm* pc )
375 {
376
377 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS, (ParallelComm*)NULL );
378 Tag pc_tag = pcomm_tag( mbImpl, true );
379 assert( 0 != pc_tag );
380
381 const EntityHandle root = 0;
382 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
383 if( MB_SUCCESS != result && MB_TAG_NOT_FOUND != result ) return -1;
384 int index = 0;
385 while( index < MAX_SHARING_PROCS && pc_array[index] )
386 index++;
387 if( index == MAX_SHARING_PROCS )
388 {
389 index = -1;
390 assert( false );
391 }
392 else
393 {
394 pc_array[index] = pc;
395 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
396 }
397 return index;
398 }
399
400 void ParallelComm::remove_pcomm( ParallelComm* pc )
401 {
402
403 std::vector< ParallelComm* > pc_array( MAX_SHARING_PROCS );
404 Tag pc_tag = pcomm_tag( mbImpl, true );
405
406 const EntityHandle root = 0;
407 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, (void*)&pc_array[0] );
408 std::vector< ParallelComm* >::iterator pc_it = std::find( pc_array.begin(), pc_array.end(), pc );
409 assert( MB_SUCCESS == result && pc_it != pc_array.end() );
410
411 if( MB_SUCCESS == result )
412 {
413 }
414
415 *pc_it = NULL;
416 mbImpl->tag_set_data( pc_tag, &root, 1, (void*)&pc_array[0] );
417 }
418
419
420
421 ErrorCode ParallelComm::assign_global_ids( EntityHandle this_set,
422 const int dimension,
423 const int start_id,
424 const bool largest_dim_only,
425 const bool parallel,
426 const bool owned_only )
427 {
428 Range entities[4];
429 ErrorCode result;
430 std::vector< unsigned char > pstatus;
431 for( int dim = 0; dim <= dimension; dim++ )
432 {
433 if( dim == 0 || !largest_dim_only || dim == dimension )
434 {
435 result = mbImpl->get_entities_by_dimension( this_set, dim, entities[dim] );MB_CHK_SET_ERR( result, "Failed to get vertices in assign_global_ids" );
436 }
437
438
439 pstatus.resize( entities[dim].size() );
440 result = mbImpl->tag_get_data( pstatus_tag(), entities[dim], &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus in assign_global_ids" );
441
442 Range dum_range;
443 Range::iterator rit;
444 unsigned int i;
445 for( rit = entities[dim].begin(), i = 0; rit != entities[dim].end(); ++rit, i++ )
446 if( pstatus[i] & PSTATUS_NOT_OWNED ) dum_range.insert( *rit );
447 entities[dim] = subtract( entities[dim], dum_range );
448 }
449
450 return assign_global_ids( entities, dimension, start_id, parallel, owned_only );
451 }
452
453
454
455 ErrorCode ParallelComm::assign_global_ids( Range entities[],
456 const int dimension,
457 const int start_id,
458 const bool parallel,
459 const bool owned_only )
460 {
461 int local_num_elements[4];
462 ErrorCode result;
463 for( int dim = 0; dim <= dimension; dim++ )
464 {
465 local_num_elements[dim] = entities[dim].size();
466 }
467
468
469 std::vector< int > num_elements( procConfig.proc_size() * 4 );
470 #ifdef MOAB_HAVE_MPI
471 if( procConfig.proc_size() > 1 && parallel )
472 {
473 int retval =
474 MPI_Allgather( local_num_elements, 4, MPI_INT, &num_elements[0], 4, MPI_INT, procConfig.proc_comm() );
475 if( 0 != retval ) return MB_FAILURE;
476 }
477 else
478 #endif
479 for( int dim = 0; dim < 4; dim++ )
480 num_elements[dim] = local_num_elements[dim];
481
482
483 int total_elems[4] = { start_id, start_id, start_id, start_id };
484
485 for( unsigned int proc = 0; proc < procConfig.proc_rank(); proc++ )
486 {
487 for( int dim = 0; dim < 4; dim++ )
488 total_elems[dim] += num_elements[4 * proc + dim];
489 }
490
491
492 Tag gid_tag = mbImpl->globalId_tag();
493
494 for( int dim = 0; dim < 4; dim++ )
495 {
496 if( entities[dim].empty() ) continue;
497 num_elements.resize( entities[dim].size() );
498 int i = 0;
499 for( Range::iterator rit = entities[dim].begin(); rit != entities[dim].end(); ++rit )
500 num_elements[i++] = total_elems[dim]++;
501
502 result = mbImpl->tag_set_data( gid_tag, entities[dim], &num_elements[0] );MB_CHK_SET_ERR( result, "Failed to set global id tag in assign_global_ids" );
503 }
504
505 if( owned_only ) return MB_SUCCESS;
506
507
508 for( int dim = 1; dim < 4; dim++ )
509 entities[0].merge( entities[dim] );
510
511 return exchange_tags( gid_tag, entities[0] );
512 }
513
514 int ParallelComm::get_buffers( int to_proc, bool* is_new )
515 {
516 int ind = -1;
517 std::vector< unsigned int >::iterator vit = std::find( buffProcs.begin(), buffProcs.end(), to_proc );
518 if( vit == buffProcs.end() )
519 {
520 assert( "shouldn't need buffer to myself" && to_proc != (int)procConfig.proc_rank() );
521 ind = buffProcs.size();
522 buffProcs.push_back( (unsigned int)to_proc );
523 localOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
524 remoteOwnedBuffs.push_back( new Buffer( INITIAL_BUFF_SIZE ) );
525 if( is_new ) *is_new = true;
526 }
527 else
528 {
529 ind = vit - buffProcs.begin();
530 if( is_new ) *is_new = false;
531 }
532 assert( ind < MAX_SHARING_PROCS );
533 return ind;
534 }
535
536 ErrorCode ParallelComm::broadcast_entities( const int from_proc,
537 Range& entities,
538 const bool adjacencies,
539 const bool tags )
540 {
541 #ifndef MOAB_HAVE_MPI
542 return MB_FAILURE;
543 #else
544
545 ErrorCode result = MB_SUCCESS;
546 int success;
547 int buff_size;
548
549 Buffer buff( INITIAL_BUFF_SIZE );
550 buff.reset_ptr( sizeof( int ) );
551 if( (int)procConfig.proc_rank() == from_proc )
552 {
553 result = add_verts( entities );MB_CHK_SET_ERR( result, "Failed to add adj vertices" );
554
555 buff.reset_ptr( sizeof( int ) );
556 result = pack_buffer( entities, adjacencies, tags, false, -1, &buff );MB_CHK_SET_ERR( result, "Failed to compute buffer size in broadcast_entities" );
557 buff.set_stored_size();
558 buff_size = buff.buff_ptr - buff.mem_ptr;
559 }
560
561 success = MPI_Bcast( &buff_size, 1, MPI_INT, from_proc, procConfig.proc_comm() );
562 if( MPI_SUCCESS != success )
563 {
564 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
565 }
566
567 if( !buff_size )
568 return MB_SUCCESS;
569
570 if( (int)procConfig.proc_rank() != from_proc ) buff.reserve( buff_size );
571
572 size_t offset = 0;
573 while( buff_size )
574 {
575 int sz = std::min( buff_size, MAX_BCAST_SIZE );
576 success = MPI_Bcast( buff.mem_ptr + offset, sz, MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
577 if( MPI_SUCCESS != success )
578 {
579 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer failed" );
580 }
581
582 offset += sz;
583 buff_size -= sz;
584 }
585
586 if( (int)procConfig.proc_rank() != from_proc )
587 {
588 std::vector< std::vector< EntityHandle > > dum1a, dum1b;
589 std::vector< std::vector< int > > dum1p;
590 std::vector< EntityHandle > dum2, dum4;
591 std::vector< unsigned int > dum3;
592 buff.reset_ptr( sizeof( int ) );
593 result = unpack_buffer( buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );MB_CHK_SET_ERR( result, "Failed to unpack buffer in broadcast_entities" );
594 std::copy( dum4.begin(), dum4.end(), range_inserter( entities ) );
595 }
596
597 return MB_SUCCESS;
598 #endif
599 }
600
601 ErrorCode ParallelComm::scatter_entities( const int from_proc,
602 std::vector< Range >& entities,
603 const bool adjacencies,
604 const bool tags )
605 {
606 #ifndef MOAB_HAVE_MPI
607 return MB_FAILURE;
608 #else
609 ErrorCode result = MB_SUCCESS;
610 int i, success, buff_size, prev_size;
611 int nProcs = (int)procConfig.proc_size();
612 int* sendCounts = new int[nProcs];
613 int* displacements = new int[nProcs];
614 sendCounts[0] = sizeof( int );
615 displacements[0] = 0;
616 Buffer buff( INITIAL_BUFF_SIZE );
617 buff.reset_ptr( sizeof( int ) );
618 buff.set_stored_size();
619 unsigned int my_proc = procConfig.proc_rank();
620
621
622 if( my_proc == (unsigned int)from_proc )
623 {
624 for( i = 1; i < nProcs; i++ )
625 {
626 prev_size = buff.buff_ptr - buff.mem_ptr;
627 buff.reset_ptr( prev_size + sizeof( int ) );
628 result = add_verts( entities[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
629
630 result = pack_buffer( entities[i], adjacencies, tags, false, -1, &buff );
631 if( MB_SUCCESS != result )
632 {
633 delete[] sendCounts;
634 delete[] displacements;
635 MB_SET_ERR( result, "Failed to pack buffer in scatter_entities" );
636 }
637
638 buff_size = buff.buff_ptr - buff.mem_ptr - prev_size;
639 *( (int*)( buff.mem_ptr + prev_size ) ) = buff_size;
640 sendCounts[i] = buff_size;
641 }
642 }
643
644
645 success = MPI_Bcast( sendCounts, nProcs, MPI_INT, from_proc, procConfig.proc_comm() );
646 if( MPI_SUCCESS != success )
647 {
648 delete[] sendCounts;
649 delete[] displacements;
650 MB_SET_ERR( MB_FAILURE, "MPI_Bcast of buffer size failed" );
651 }
652
653 for( i = 1; i < nProcs; i++ )
654 {
655 displacements[i] = displacements[i - 1] + sendCounts[i - 1];
656 }
657
658 Buffer rec_buff;
659 rec_buff.reserve( sendCounts[my_proc] );
660
661
662 success = MPI_Scatterv( buff.mem_ptr, sendCounts, displacements, MPI_UNSIGNED_CHAR, rec_buff.mem_ptr,
663 sendCounts[my_proc], MPI_UNSIGNED_CHAR, from_proc, procConfig.proc_comm() );
664
665 if( MPI_SUCCESS != success )
666 {
667 delete[] sendCounts;
668 delete[] displacements;
669 MB_SET_ERR( MB_FAILURE, "MPI_Scatterv of buffer failed" );
670 }
671
672
673 if( my_proc != (unsigned int)from_proc )
674 {
675 std::vector< std::vector< EntityHandle > > dum1a, dum1b;
676 std::vector< std::vector< int > > dum1p;
677 std::vector< EntityHandle > dum2, dum4;
678 std::vector< unsigned int > dum3;
679 rec_buff.reset_ptr( sizeof( int ) );
680 result = unpack_buffer( rec_buff.buff_ptr, false, from_proc, -1, dum1a, dum1b, dum1p, dum2, dum2, dum3, dum4 );
681 if( MB_SUCCESS != result )
682 {
683 delete[] sendCounts;
684 delete[] displacements;
685 MB_SET_ERR( result, "Failed to unpack buffer in scatter_entities" );
686 }
687
688 std::copy( dum4.begin(), dum4.end(), range_inserter( entities[my_proc] ) );
689 }
690
691 delete[] sendCounts;
692 delete[] displacements;
693
694 return MB_SUCCESS;
695 #endif
696 }
697
698 ErrorCode ParallelComm::send_entities( const int to_proc,
699 Range& orig_ents,
700 const bool adjs,
701 const bool tags,
702 const bool store_remote_handles,
703 const bool is_iface,
704 Range& ,
705 int& incoming1,
706 int& incoming2,
707 TupleList& entprocs,
708 std::vector< MPI_Request >& recv_remoteh_reqs,
709 bool )
710 {
711 #ifndef MOAB_HAVE_MPI
712 return MB_FAILURE;
713 #else
714
715 int ind = get_buffers( to_proc );
716 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
717
718
719 ErrorCode result = add_verts( orig_ents );MB_CHK_SET_ERR( result, "Failed to add verts in send_entities" );
720
721
722 Range tmp_range;
723 result = filter_pstatus( orig_ents, PSTATUS_SHARED, PSTATUS_AND, to_proc, &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
724 if( !tmp_range.empty() )
725 {
726 orig_ents = subtract( orig_ents, tmp_range );
727 }
728
729 result = pack_buffer( orig_ents, adjs, tags, store_remote_handles, to_proc, localOwnedBuffs[ind], &entprocs );MB_CHK_SET_ERR( result, "Failed to pack buffer in send_entities" );
730
731
732 result = send_buffer( to_proc, localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind], recvReqs[2 * ind + 1],
733 (int*)( remoteOwnedBuffs[ind]->mem_ptr ),
734
735 incoming1, MB_MESG_REMOTEH_SIZE,
736 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind] : NULL ),
737 &recv_remoteh_reqs[2 * ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to send buffer" );
738
739 return MB_SUCCESS;
740 #endif
741 }
742
743 ErrorCode ParallelComm::send_entities( std::vector< unsigned int >& send_procs,
744 std::vector< Range* >& send_ents,
745 int& incoming1,
746 int& incoming2,
747 const bool store_remote_handles )
748 {
749 #ifdef MOAB_HAVE_MPE
750 if( myDebug->get_verbosity() == 2 )
751 {
752 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_entities." );
753 }
754 #endif
755 myDebug->tprintf( 1, "Entering send_entities\n" );
756 if( myDebug->get_verbosity() == 4 )
757 {
758 msgs.clear();
759 msgs.reserve( MAX_SHARING_PROCS );
760 }
761
762 unsigned int i;
763 int ind;
764 ErrorCode result = MB_SUCCESS;
765
766
767 unsigned int n_proc = send_procs.size();
768 for( i = 0; i < n_proc; i++ )
769 {
770 ind = get_buffers( send_procs[i] );
771 result = add_verts( *send_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
772
773
774 Range tmp_range;
775 result = filter_pstatus( *send_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
776 if( !tmp_range.empty() )
777 {
778 *send_ents[i] = subtract( *send_ents[i], tmp_range );
779 }
780 }
781
782
783
784
785
786 Range allsent, tmp_range;
787 int npairs = 0;
788 TupleList entprocs;
789 for( i = 0; i < n_proc; i++ )
790 {
791 int n_ents = send_ents[i]->size();
792 if( n_ents > 0 )
793 {
794 npairs += n_ents;
795 allsent.merge( *send_ents[i] );
796 }
797 }
798
799
800 entprocs.initialize( 1, 0, 1, 0, npairs );
801 entprocs.enableWriteAccess();
802
803
804 for( i = 0; i < n_proc; i++ )
805 {
806 for( Range::iterator rit = send_ents[i]->begin(); rit != send_ents[i]->end(); ++rit )
807 {
808 entprocs.vi_wr[entprocs.get_n()] = send_procs[i];
809 entprocs.vul_wr[entprocs.get_n()] = *rit;
810 entprocs.inc_n();
811 }
812 }
813
814
815 moab::TupleList::buffer sort_buffer;
816 sort_buffer.buffer_init( npairs );
817 entprocs.sort( 1, &sort_buffer );
818 entprocs.disableWriteAccess();
819 sort_buffer.reset();
820
821 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
822 (unsigned long)allsent.size() );
823
824
825
826
827 for( i = 0; i < n_proc; i++ )
828 {
829 if( send_ents[i]->size() > 0 )
830 {
831 ind = get_buffers( send_procs[i] );
832 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", send_ents[i]->compactness(),
833 (unsigned long)send_ents[i]->size() );
834
835 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
836 result = pack_buffer( *send_ents[i], false, true, store_remote_handles, buffProcs[ind],
837 localOwnedBuffs[ind], &entprocs, &allsent );
838
839 if( myDebug->get_verbosity() == 4 )
840 {
841 msgs.resize( msgs.size() + 1 );
842 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
843 }
844
845
846 result = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[2 * ind],
847 recvReqs[2 * ind + 1], &ackbuff, incoming1, MB_MESG_REMOTEH_SIZE,
848 ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recvRemotehReqs[2 * ind],
849 &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost send" );
850 }
851 }
852 entprocs.reset();
853
854 #ifdef MOAB_HAVE_MPE
855 if( myDebug->get_verbosity() == 2 )
856 {
857 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_entities." );
858 }
859 #endif
860
861 return MB_SUCCESS;
862 }
863
864
865
866
867 void print_buff( unsigned char* ch, int size )
868 {
869 for( int i = 0; i < size; i++ )
870 std::cout << ch[i];
871 std::cout << "\n";
872 }
873 ErrorCode ParallelComm::send_recv_entities( std::vector< int >& send_procs,
874 std::vector< std::vector< int > >& msgsizes,
875 std::vector< std::vector< EntityHandle > >& senddata,
876 std::vector< std::vector< EntityHandle > >& recvdata )
877 {
878 #ifdef USE_MPE
879 if( myDebug->get_verbosity() == 2 )
880 {
881 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting send_recv_entities." );
882 }
883 #endif
884 myDebug->tprintf( 1, "Entering send_recv_entities\n" );
885 if( myDebug->get_verbosity() == 4 )
886 {
887 msgs.clear();
888 msgs.reserve( MAX_SHARING_PROCS );
889 }
890
891
892 int i, ind, success;
893 ErrorCode error = MB_SUCCESS;
894
895
896
897
898
899
900
901 reset_all_buffers();
902 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
903 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
904 int ack_buff;
905 int incoming = 0;
906
907 std::vector< unsigned int >::iterator sit;
908
909 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
910 {
911 incoming++;
912 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
913 MB_MESG_ENTS_SIZE, incoming );
914
915 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
916 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
917 if( success != MPI_SUCCESS )
918 {
919 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in send_recv_entities" );
920 }
921 }
922
923
924 for( i = 0; i < (int)send_procs.size(); i++ )
925 {
926
927 ind = get_buffers( send_procs[i] );
928 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
929
930 int buff_size = msgsizes[i].size() * sizeof( int ) + senddata[i].size() * sizeof( EntityHandle );
931 localOwnedBuffs[ind]->check_space( buff_size );
932
933
934 std::vector< int > msg;
935 msg.insert( msg.end(), msgsizes[i].begin(), msgsizes[i].end() );
936 PACK_INTS( localOwnedBuffs[ind]->buff_ptr, &msg[0], msg.size() );
937
938 std::vector< EntityHandle > entities;
939 entities.insert( entities.end(), senddata[i].begin(), senddata[i].end() );
940 PACK_EH( localOwnedBuffs[ind]->buff_ptr, &entities[0], entities.size() );
941 localOwnedBuffs[ind]->set_stored_size();
942
943 if( myDebug->get_verbosity() == 4 )
944 {
945 msgs.resize( msgs.size() + 1 );
946 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
947 }
948
949
950 error = send_buffer( send_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind],
951 recv_ent_reqs[3 * ind + 2], &ack_buff, incoming );MB_CHK_SET_ERR( error, "Failed to Isend in send_recv_entities" );
952 }
953
954
955
956
957
958 while( incoming )
959 {
960
961 MPI_Status status;
962 int index_in_recv_requests;
963
964 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
965 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &index_in_recv_requests, &status );
966 if( MPI_SUCCESS != success )
967 {
968 MB_SET_ERR( MB_FAILURE, "Failed in waitany in send_recv_entities" );
969 }
970
971
972 ind = index_in_recv_requests / 3;
973
974 PRINT_DEBUG_RECD( status );
975
976
977 incoming--;
978
979 bool done = false;
980
981 error = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind],
982 recv_ent_reqs[3 * ind + 1],
983 recv_ent_reqs[3 * ind + 2],
984
985 incoming, localOwnedBuffs[ind],
986 sendReqs[3 * ind + 1],
987 sendReqs[3 * ind + 2],
988 done );MB_CHK_SET_ERR( error, "Failed to resize recv buffer" );
989
990 if( done )
991 {
992 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
993
994 int from_proc = status.MPI_SOURCE;
995 int idx = std::find( send_procs.begin(), send_procs.end(), from_proc ) - send_procs.begin();
996
997 int msg = msgsizes[idx].size();
998 std::vector< int > recvmsg( msg );
999 int ndata = senddata[idx].size();
1000 std::vector< EntityHandle > dum_vec( ndata );
1001
1002 UNPACK_INTS( remoteOwnedBuffs[ind]->buff_ptr, &recvmsg[0], msg );
1003 UNPACK_EH( remoteOwnedBuffs[ind]->buff_ptr, &dum_vec[0], ndata );
1004
1005 recvdata[idx].insert( recvdata[idx].end(), dum_vec.begin(), dum_vec.end() );
1006 }
1007 }
1008
1009 #ifdef USE_MPE
1010 if( myDebug->get_verbosity() == 2 )
1011 {
1012 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending send_recv_entities." );
1013 }
1014 #endif
1015
1016 return MB_SUCCESS;
1017 }
1018
1019 ErrorCode ParallelComm::update_remote_data( EntityHandle entity,
1020 std::vector< int >& procs,
1021 std::vector< EntityHandle >& handles )
1022 {
1023 ErrorCode error;
1024 unsigned char pstatus = PSTATUS_INTERFACE;
1025
1026 int procmin = *std::min_element( procs.begin(), procs.end() );
1027
1028 if( (int)rank() > procmin )
1029 pstatus |= PSTATUS_NOT_OWNED;
1030 else
1031 procmin = rank();
1032
1033
1034
1035
1036
1037
1038
1039 if( (int)procs.size() > 1 )
1040 {
1041 procs.push_back( rank() );
1042 handles.push_back( entity );
1043
1044 int idx = std::find( procs.begin(), procs.end(), procmin ) - procs.begin();
1045
1046 std::iter_swap( procs.begin(), procs.begin() + idx );
1047 std::iter_swap( handles.begin(), handles.begin() + idx );
1048
1049
1050
1051
1052
1053
1054 }
1055
1056
1057
1058
1059 error = update_remote_data( entity, &procs[0], &handles[0], procs.size(), pstatus );MB_CHK_ERR( error );
1060
1061 return MB_SUCCESS;
1062 }
1063
1064 ErrorCode ParallelComm::get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc )
1065 {
1066 ErrorCode error;
1067 std::vector< EntityHandle > newents;
1068 error = get_remote_handles( true, local_vec, rem_vec, num_ents, to_proc, newents );MB_CHK_ERR( error );
1069
1070 return MB_SUCCESS;
1071 }
1072
1073
1074
1075 ErrorCode ParallelComm::recv_entities( const int from_proc,
1076 const bool store_remote_handles,
1077 const bool is_iface,
1078 Range& final_ents,
1079 int& incoming1,
1080 int& incoming2,
1081 std::vector< std::vector< EntityHandle > >& L1hloc,
1082 std::vector< std::vector< EntityHandle > >& L1hrem,
1083 std::vector< std::vector< int > >& L1p,
1084 std::vector< EntityHandle >& L2hloc,
1085 std::vector< EntityHandle >& L2hrem,
1086 std::vector< unsigned int >& L2p,
1087 std::vector< MPI_Request >& recv_remoteh_reqs,
1088 bool )
1089 {
1090 #ifndef MOAB_HAVE_MPI
1091 return MB_FAILURE;
1092 #else
1093
1094 int ind1 = get_buffers( from_proc );
1095 incoming1++;
1096 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE,
1097 MB_MESG_ENTS_SIZE, incoming1 );
1098 int success = MPI_Irecv( remoteOwnedBuffs[ind1]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc,
1099 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind1] );
1100 if( success != MPI_SUCCESS )
1101 {
1102 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
1103 }
1104
1105
1106 return recv_messages( from_proc, store_remote_handles, is_iface, final_ents, incoming1, incoming2, L1hloc, L1hrem,
1107 L1p, L2hloc, L2hrem, L2p, recv_remoteh_reqs );
1108 #endif
1109 }
1110
1111 ErrorCode ParallelComm::recv_entities( std::set< unsigned int >& recv_procs,
1112 int incoming1,
1113 int incoming2,
1114 const bool store_remote_handles,
1115 const bool migrate )
1116 {
1117
1118
1119
1120
1121 int success, ind, i;
1122 ErrorCode result;
1123 MPI_Status status;
1124 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
1125 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
1126 std::vector< std::vector< int > > L1p( buffProcs.size() );
1127 std::vector< EntityHandle > L2hloc, L2hrem;
1128 std::vector< unsigned int > L2p;
1129 std::vector< EntityHandle > new_ents;
1130
1131 while( incoming1 )
1132 {
1133
1134
1135 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
1136
1137 success = MPI_Waitany( 2 * buffProcs.size(), &recvReqs[0], &ind, &status );
1138 if( MPI_SUCCESS != success )
1139 {
1140 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
1141 }
1142
1143 PRINT_DEBUG_RECD( status );
1144
1145
1146 incoming1--;
1147 bool done = false;
1148
1149
1150 unsigned int base_ind = 2 * ( ind / 2 );
1151 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 2], recvReqs[ind], recvReqs[ind + 1],
1152 incoming1, localOwnedBuffs[ind / 2], sendReqs[base_ind], sendReqs[base_ind + 1], done,
1153 ( store_remote_handles ? localOwnedBuffs[ind / 2] : NULL ), MB_MESG_REMOTEH_SIZE,
1154 &recvRemotehReqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
1155
1156 if( done )
1157 {
1158 if( myDebug->get_verbosity() == 4 )
1159 {
1160 msgs.resize( msgs.size() + 1 );
1161 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 2] );
1162 }
1163
1164
1165 remoteOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
1166 result = unpack_buffer( remoteOwnedBuffs[ind / 2]->buff_ptr, store_remote_handles, buffProcs[ind / 2],
1167 ind / 2, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true );
1168 if( MB_SUCCESS != result )
1169 {
1170 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
1171 print_buffer( remoteOwnedBuffs[ind / 2]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 2], false );
1172 return result;
1173 }
1174
1175 if( recvReqs.size() != 2 * buffProcs.size() )
1176 {
1177
1178 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
1179 for( i = recvReqs.size(); i < (int)( 2 * buffProcs.size() ); i += 2 )
1180 {
1181 localOwnedBuffs[i / 2]->reset_buffer();
1182 incoming2++;
1183 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 2], localOwnedBuffs[i / 2]->mem_ptr,
1184 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
1185 success = MPI_Irecv( localOwnedBuffs[i / 2]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
1186 buffProcs[i / 2], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
1187 &recvRemotehReqs[i] );
1188 if( success != MPI_SUCCESS )
1189 {
1190 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
1191 }
1192 }
1193 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
1194 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
1195 }
1196 }
1197 }
1198
1199
1200 result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" );
1201 if( migrate )
1202 {
1203
1204
1205 }
1206
1207
1208 if( recvReqs.size() != 2 * buffProcs.size() )
1209 {
1210
1211 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" );
1212 }
1213
1214 #ifdef MOAB_HAVE_MPE
1215 if( myDebug->get_verbosity() == 2 )
1216 {
1217 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending recv entities." );
1218 }
1219 #endif
1220
1221
1222
1223
1224 std::set< unsigned int >::iterator it = recv_procs.begin();
1225 std::set< unsigned int >::iterator eit = recv_procs.end();
1226 for( ; it != eit; ++it )
1227 {
1228 ind = get_buffers( *it );
1229
1230 remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
1231
1232 result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
1233 remoteOwnedBuffs[ind]->set_stored_size();
1234
1235 if( myDebug->get_verbosity() == 4 )
1236 {
1237 msgs.resize( msgs.size() + 1 );
1238 msgs.back() = new Buffer( *remoteOwnedBuffs[ind] );
1239 }
1240 result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[2 * ind],
1241 recvRemotehReqs[2 * ind + 1], &ackbuff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
1242 }
1243
1244
1245
1246
1247 while( incoming2 )
1248 {
1249 PRINT_DEBUG_WAITANY( recvRemotehReqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
1250 success = MPI_Waitany( 2 * buffProcs.size(), &recvRemotehReqs[0], &ind, &status );
1251 if( MPI_SUCCESS != success )
1252 {
1253 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
1254 }
1255
1256
1257 incoming2--;
1258
1259 PRINT_DEBUG_RECD( status );
1260 bool done = false;
1261 unsigned int base_ind = 2 * ( ind / 2 );
1262 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 2], recvRemotehReqs[ind],
1263 recvRemotehReqs[ind + 1], incoming2, remoteOwnedBuffs[ind / 2], sendReqs[base_ind],
1264 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
1265 if( done )
1266 {
1267
1268 if( myDebug->get_verbosity() == 4 )
1269 {
1270 msgs.resize( msgs.size() + 1 );
1271 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
1272 }
1273
1274 localOwnedBuffs[ind / 2]->reset_ptr( sizeof( int ) );
1275 result =
1276 unpack_remote_handles( buffProcs[ind / 2], localOwnedBuffs[ind / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
1277 }
1278 }
1279
1280 #ifdef MOAB_HAVE_MPE
1281 if( myDebug->get_verbosity() == 2 )
1282 {
1283 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
1284 MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending recv entities (still doing checks)." );
1285 }
1286 #endif
1287 myDebug->tprintf( 1, "Exiting recv_entities.\n" );
1288
1289 return MB_SUCCESS;
1290 }
1291
1292 ErrorCode ParallelComm::recv_messages( const int from_proc,
1293 const bool store_remote_handles,
1294 const bool is_iface,
1295 Range& final_ents,
1296 int& incoming1,
1297 int& incoming2,
1298 std::vector< std::vector< EntityHandle > >& L1hloc,
1299 std::vector< std::vector< EntityHandle > >& L1hrem,
1300 std::vector< std::vector< int > >& L1p,
1301 std::vector< EntityHandle >& L2hloc,
1302 std::vector< EntityHandle >& L2hrem,
1303 std::vector< unsigned int >& L2p,
1304 std::vector< MPI_Request >& recv_remoteh_reqs )
1305 {
1306 #ifndef MOAB_HAVE_MPI
1307 return MB_FAILURE;
1308 #else
1309 MPI_Status status;
1310 ErrorCode result;
1311 int ind1 = get_buffers( from_proc );
1312 int success, ind2;
1313 std::vector< EntityHandle > new_ents;
1314
1315
1316 while( incoming1 )
1317 {
1318 PRINT_DEBUG_WAITANY( recvReqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
1319 success = MPI_Waitany( 2, &recvReqs[2 * ind1], &ind2, &status );
1320 if( MPI_SUCCESS != success )
1321 {
1322 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_messages" );
1323 }
1324
1325 PRINT_DEBUG_RECD( status );
1326
1327
1328 incoming1--;
1329 bool done = false;
1330
1331
1332 ind2 += 2 * ind1;
1333 unsigned int base_ind = 2 * ( ind2 / 2 );
1334
1335 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind2 / 2],
1336
1337 recvReqs[ind2], recvReqs[ind2 + 1], incoming1, localOwnedBuffs[ind2 / 2],
1338 sendReqs[base_ind], sendReqs[base_ind + 1], done,
1339 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind2 / 2] : NULL ),
1340 MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
1341
1342 if( done )
1343 {
1344
1345 remoteOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
1346 result = unpack_buffer( remoteOwnedBuffs[ind2 / 2]->buff_ptr, store_remote_handles, from_proc, ind2 / 2,
1347 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );MB_CHK_SET_ERR( result, "Failed to unpack buffer in recev_messages" );
1348
1349 std::copy( new_ents.begin(), new_ents.end(), range_inserter( final_ents ) );
1350
1351
1352
1353 remoteOwnedBuffs[ind2 / 2]->reset_buffer( sizeof( int ) );
1354
1355 result = pack_remote_handles( L1hloc[ind2 / 2], L1hrem[ind2 / 2], L1p[ind2 / 2], from_proc,
1356 remoteOwnedBuffs[ind2 / 2] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
1357 remoteOwnedBuffs[ind2 / 2]->set_stored_size();
1358
1359 result = send_buffer( buffProcs[ind2 / 2], remoteOwnedBuffs[ind2 / 2], MB_MESG_REMOTEH_SIZE, sendReqs[ind2],
1360 recv_remoteh_reqs[ind2 + 1], (int*)( localOwnedBuffs[ind2 / 2]->mem_ptr ),
1361
1362 incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
1363 }
1364 }
1365
1366 return MB_SUCCESS;
1367 #endif
1368 }
1369
1370 ErrorCode ParallelComm::recv_remote_handle_messages( const int from_proc,
1371 int& incoming2,
1372 std::vector< EntityHandle >& L2hloc,
1373 std::vector< EntityHandle >& L2hrem,
1374 std::vector< unsigned int >& L2p,
1375 std::vector< MPI_Request >& recv_remoteh_reqs )
1376 {
1377 #ifndef MOAB_HAVE_MPI
1378 return MB_FAILURE;
1379 #else
1380 MPI_Status status;
1381 ErrorCode result;
1382 int ind1 = get_buffers( from_proc );
1383 int success, ind2;
1384
1385 while( incoming2 )
1386 {
1387 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
1388 success = MPI_Waitany( 2, &recv_remoteh_reqs[2 * ind1], &ind2, &status );
1389 if( MPI_SUCCESS != success )
1390 {
1391 MB_SET_ERR( MB_FAILURE, "Failed in waitany in recv_remote_handle_messages" );
1392 }
1393
1394
1395 incoming2--;
1396
1397 PRINT_DEBUG_RECD( status );
1398
1399 bool done = false;
1400 ind2 += 2 * ind1;
1401 unsigned int base_ind = 2 * ( ind2 / 2 );
1402 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind2 / 2], recv_remoteh_reqs[ind2],
1403 recv_remoteh_reqs[ind2 + 1], incoming2, remoteOwnedBuffs[ind2 / 2], sendReqs[base_ind],
1404 sendReqs[base_ind + 1], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
1405 if( done )
1406 {
1407
1408 localOwnedBuffs[ind2 / 2]->reset_ptr( sizeof( int ) );
1409 result =
1410 unpack_remote_handles( buffProcs[ind2 / 2], localOwnedBuffs[ind2 / 2]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
1411 }
1412 }
1413
1414 return MB_SUCCESS;
1415 #endif
1416 }
1417
1418 ErrorCode ParallelComm::pack_buffer( Range& orig_ents,
1419 const bool ,
1420 const bool tags,
1421 const bool store_remote_handles,
1422 const int to_proc,
1423 Buffer* buff,
1424 TupleList* entprocs,
1425 Range* allsent )
1426 {
1427
1428
1429
1430
1431
1432 ErrorCode result;
1433
1434 Range set_range;
1435 std::vector< Tag > all_tags;
1436 std::vector< Range > tag_ranges;
1437
1438 Range::const_iterator rit;
1439
1440
1441 result = pack_entities( orig_ents, buff, store_remote_handles, to_proc, false, entprocs, allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
1442
1443
1444 result = pack_sets( orig_ents, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing sets (count) failed" );
1445
1446
1447 Range final_ents;
1448 if( tags )
1449 {
1450 result = get_tag_send_list( orig_ents, all_tags, tag_ranges );MB_CHK_SET_ERR( result, "Failed to get tagged entities" );
1451 result = pack_tags( orig_ents, all_tags, all_tags, tag_ranges, buff, store_remote_handles, to_proc );MB_CHK_SET_ERR( result, "Packing tags (count) failed" );
1452 }
1453 else
1454 {
1455 buff->check_space( sizeof( int ) );
1456 PACK_INT( buff->buff_ptr, 0 );
1457 buff->set_stored_size();
1458 }
1459
1460 return result;
1461 }
1462
1463 ErrorCode ParallelComm::unpack_buffer( unsigned char* buff_ptr,
1464 const bool store_remote_handles,
1465 const int from_proc,
1466 const int ind,
1467 std::vector< std::vector< EntityHandle > >& L1hloc,
1468 std::vector< std::vector< EntityHandle > >& L1hrem,
1469 std::vector< std::vector< int > >& L1p,
1470 std::vector< EntityHandle >& L2hloc,
1471 std::vector< EntityHandle >& L2hrem,
1472 std::vector< unsigned int >& L2p,
1473 std::vector< EntityHandle >& new_ents,
1474 const bool created_iface )
1475 {
1476 unsigned char* tmp_buff = buff_ptr;
1477 ErrorCode result;
1478 result = unpack_entities( buff_ptr, store_remote_handles, ind, false, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p,
1479 new_ents, created_iface );MB_CHK_SET_ERR( result, "Unpacking entities failed" );
1480 if( myDebug->get_verbosity() == 3 )
1481 {
1482 myDebug->tprintf( 4, "unpack_entities buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
1483 tmp_buff = buff_ptr;
1484 }
1485 result = unpack_sets( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking sets failed" );
1486 if( myDebug->get_verbosity() == 3 )
1487 {
1488 myDebug->tprintf( 4, "unpack_sets buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
1489 tmp_buff = buff_ptr;
1490 }
1491 result = unpack_tags( buff_ptr, new_ents, store_remote_handles, from_proc );MB_CHK_SET_ERR( result, "Unpacking tags failed" );
1492 if( myDebug->get_verbosity() == 3 )
1493 {
1494 myDebug->tprintf( 4, "unpack_tags buffer space: %ld bytes.\n", (long int)( buff_ptr - tmp_buff ) );
1495
1496 }
1497
1498 if( myDebug->get_verbosity() == 3 ) myDebug->print( 4, "\n" );
1499
1500 return MB_SUCCESS;
1501 }
1502
1503 int ParallelComm::estimate_ents_buffer_size( Range& entities, const bool store_remote_handles )
1504 {
1505 int buff_size = 0;
1506 std::vector< EntityHandle > dum_connect_vec;
1507 const EntityHandle* connect;
1508 int num_connect;
1509
1510 int num_verts = entities.num_of_type( MBVERTEX );
1511
1512 buff_size += 2 * sizeof( int ) + 3 * sizeof( double ) * num_verts;
1513 if( store_remote_handles ) buff_size += sizeof( EntityHandle ) * num_verts;
1514
1515
1516 for( EntityType t = MBEDGE; t < MBENTITYSET; t++ )
1517 {
1518 const Range::iterator rit = entities.lower_bound( t );
1519 if( TYPE_FROM_HANDLE( *rit ) != t ) continue;
1520
1521 ErrorCode result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect_vec );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get connectivity to estimate buffer size", -1 );
1522
1523
1524 buff_size += 3 * sizeof( int );
1525 int num_ents = entities.num_of_type( t );
1526
1527 buff_size += ( num_connect + 1 ) * sizeof( EntityHandle ) * num_ents;
1528 }
1529
1530
1531 buff_size += sizeof( int );
1532
1533 return buff_size;
1534 }
1535
1536 int ParallelComm::estimate_sets_buffer_size( Range& entities, const bool )
1537 {
1538
1539 int buff_size = sizeof( int );
1540
1541
1542 Range::iterator rit = entities.lower_bound( MBENTITYSET );
1543 ErrorCode result;
1544
1545 for( ; rit != entities.end(); ++rit )
1546 {
1547 unsigned int options;
1548 result = mbImpl->get_meshset_options( *rit, options );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get meshset options", -1 );
1549
1550 buff_size += sizeof( int );
1551
1552 Range set_range;
1553 if( options & MESHSET_SET )
1554 {
1555
1556 result = mbImpl->get_entities_by_handle( *rit, set_range );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get set entities", -1 );
1557
1558
1559 buff_size += RANGE_SIZE( set_range );
1560 }
1561 else if( options & MESHSET_ORDERED )
1562 {
1563
1564 int num_ents;
1565 result = mbImpl->get_number_entities_by_handle( *rit, num_ents );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get number entities in ordered set", -1 );
1566
1567
1568 buff_size += sizeof( EntityHandle ) * num_ents + sizeof( int );
1569 }
1570
1571
1572 int num_par, num_ch;
1573 result = mbImpl->num_child_meshsets( *rit, &num_ch );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num children", -1 );
1574 result = mbImpl->num_parent_meshsets( *rit, &num_par );MB_CHK_SET_ERR_RET_VAL( result, "Failed to get num parents", -1 );
1575
1576 buff_size += ( num_ch + num_par ) * sizeof( EntityHandle ) + 2 * sizeof( int );
1577 }
1578
1579 return buff_size;
1580 }
1581
1582 ErrorCode ParallelComm::pack_entities( Range& entities,
1583 Buffer* buff,
1584 const bool store_remote_handles,
1585 const int to_proc,
1586 const bool ,
1587 TupleList* entprocs,
1588 Range* )
1589 {
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599 int buff_size = estimate_ents_buffer_size( entities, store_remote_handles );
1600 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate ents buffer size" );
1601 buff->check_space( buff_size );
1602 myDebug->tprintf( 3, "estimate buffer size for %d entities: %d \n", (int)entities.size(), buff_size );
1603
1604 unsigned int num_ents;
1605 ErrorCode result;
1606
1607 std::vector< EntityHandle > entities_vec( entities.size() );
1608 std::copy( entities.begin(), entities.end(), entities_vec.begin() );
1609
1610
1611
1612 if( store_remote_handles )
1613 {
1614
1615
1616 buff->check_space( sizeof( int ) + ( 5 * sizeof( int ) + sizeof( EntityHandle ) ) * entities.size() );
1617
1618
1619 PACK_INT( buff->buff_ptr, entities.size() );
1620
1621 Range::iterator rit;
1622
1623
1624 std::vector< int > sharedp_vals( entities.size() );
1625 result = mbImpl->tag_get_data( sharedp_tag(), entities, &sharedp_vals[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
1626 std::vector< char > pstatus_vals( entities.size() );
1627 result = mbImpl->tag_get_data( pstatus_tag(), entities, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
1628
1629 unsigned int i;
1630 int tmp_procs[MAX_SHARING_PROCS];
1631 EntityHandle tmp_handles[MAX_SHARING_PROCS];
1632 std::set< unsigned int > dumprocs;
1633
1634
1635 for( rit = entities.begin(), i = 0; rit != entities.end(); ++rit, i++ )
1636 {
1637 unsigned int ind =
1638 std::lower_bound( entprocs->vul_rd, entprocs->vul_rd + entprocs->get_n(), *rit ) - entprocs->vul_rd;
1639 assert( ind < entprocs->get_n() );
1640
1641 while( ind < entprocs->get_n() && entprocs->vul_rd[ind] == *rit )
1642 dumprocs.insert( entprocs->vi_rd[ind++] );
1643
1644 result = build_sharedhps_list( *rit, pstatus_vals[i], sharedp_vals[i], dumprocs, num_ents, tmp_procs,
1645 tmp_handles );MB_CHK_SET_ERR( result, "Failed to build sharedhps" );
1646
1647 dumprocs.clear();
1648
1649
1650 buff->check_space( ( num_ents + 1 ) * sizeof( int ) + num_ents * sizeof( EntityHandle ) );
1651 PACK_INT( buff->buff_ptr, num_ents );
1652 PACK_INTS( buff->buff_ptr, tmp_procs, num_ents );
1653 PACK_EH( buff->buff_ptr, tmp_handles, num_ents );
1654
1655 #ifndef NDEBUG
1656
1657 unsigned int dp = 0;
1658 for( ; dp < MAX_SHARING_PROCS && -1 != tmp_procs[dp]; dp++ )
1659 dumprocs.insert( tmp_procs[dp] );
1660 assert( dumprocs.size() == dp );
1661 dumprocs.clear();
1662 #endif
1663 }
1664 }
1665
1666
1667 Range these_ents = entities.subset_by_type( MBVERTEX );
1668 num_ents = these_ents.size();
1669
1670 if( num_ents )
1671 {
1672 buff_size = 2 * sizeof( int ) + 3 * num_ents * sizeof( double );
1673 buff->check_space( buff_size );
1674
1675
1676 PACK_INT( buff->buff_ptr, ( (int)MBVERTEX ) );
1677 PACK_INT( buff->buff_ptr, ( (int)num_ents ) );
1678
1679 std::vector< double > tmp_coords( 3 * num_ents );
1680 result = mbImpl->get_coords( these_ents, &tmp_coords[0] );MB_CHK_SET_ERR( result, "Failed to get vertex coordinates" );
1681 PACK_DBLS( buff->buff_ptr, &tmp_coords[0], 3 * num_ents );
1682
1683 myDebug->tprintf( 4, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
1684 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
1685 }
1686
1687
1688 Range::iterator start_rit = entities.find( *these_ents.rbegin() );
1689 ++start_rit;
1690 int last_nodes = -1;
1691 EntityType last_type = MBMAXTYPE;
1692 these_ents.clear();
1693 Range::iterator end_rit = start_rit;
1694 EntitySequence* seq;
1695 ElementSequence* eseq;
1696
1697 while( start_rit != entities.end() || !these_ents.empty() )
1698 {
1699
1700
1701
1702
1703
1704
1705
1706 eseq = NULL;
1707 if( start_rit != entities.end() )
1708 {
1709 result = sequenceManager->find( *start_rit, seq );MB_CHK_SET_ERR( result, "Failed to find entity sequence" );
1710 if( NULL == seq ) return MB_FAILURE;
1711 eseq = dynamic_cast< ElementSequence* >( seq );
1712 }
1713
1714
1715 if( !these_ents.empty() &&
1716 ( !eseq || eseq->type() != last_type || last_nodes != (int)eseq->nodes_per_element() ) )
1717 {
1718 result = pack_entity_seq( last_nodes, store_remote_handles, to_proc, these_ents, entities_vec, buff );MB_CHK_SET_ERR( result, "Failed to pack entities from a sequence" );
1719 these_ents.clear();
1720 }
1721
1722 if( eseq )
1723 {
1724
1725
1726 end_rit = entities.lower_bound( start_rit, entities.end(), eseq->end_handle() + 1 );
1727
1728
1729 std::copy( start_rit, end_rit, range_inserter( these_ents ) );
1730
1731 last_type = eseq->type();
1732 last_nodes = eseq->nodes_per_element();
1733 }
1734 else if( start_rit != entities.end() && TYPE_FROM_HANDLE( *start_rit ) == MBENTITYSET )
1735 break;
1736
1737 start_rit = end_rit;
1738 }
1739
1740
1741 buff->check_space( sizeof( int ) );
1742 PACK_INT( buff->buff_ptr, ( (int)MBMAXTYPE ) );
1743
1744 buff->set_stored_size();
1745 return MB_SUCCESS;
1746 }
1747
1748 ErrorCode ParallelComm::build_sharedhps_list( const EntityHandle entity,
1749 const unsigned char pstatus,
1750 const int
1751 #ifndef NDEBUG
1752 sharedp
1753 #endif
1754 ,
1755 const std::set< unsigned int >& procs,
1756 unsigned int& num_ents,
1757 int* tmp_procs,
1758 EntityHandle* tmp_handles )
1759 {
1760 num_ents = 0;
1761 unsigned char pstat;
1762 ErrorCode result = get_sharing_data( entity, tmp_procs, tmp_handles, pstat, num_ents );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
1763 assert( pstat == pstatus );
1764
1765
1766
1767 if( pstatus & PSTATUS_MULTISHARED )
1768 {
1769 }
1770 else if( pstatus & PSTATUS_NOT_OWNED )
1771 {
1772
1773
1774 assert( "If not owned, I should be shared too" && pstatus & PSTATUS_SHARED && 1 == num_ents );
1775 tmp_procs[1] = procConfig.proc_rank();
1776 tmp_handles[1] = entity;
1777 num_ents = 2;
1778 }
1779 else if( pstatus & PSTATUS_SHARED )
1780 {
1781
1782 assert( "shared and owned, should be only 1 sharing proc" && 1 == num_ents );
1783 tmp_procs[1] = tmp_procs[0];
1784 tmp_procs[0] = procConfig.proc_rank();
1785 tmp_handles[1] = tmp_handles[0];
1786 tmp_handles[0] = entity;
1787 num_ents = 2;
1788 }
1789 else
1790 {
1791
1792 tmp_procs[0] = procConfig.proc_rank();
1793 tmp_handles[0] = entity;
1794 num_ents = 1;
1795 }
1796
1797 #ifndef NDEBUG
1798 int tmp_ps = num_ents;
1799 #endif
1800
1801
1802 for( std::set< unsigned int >::iterator sit = procs.begin(); sit != procs.end(); ++sit )
1803 {
1804 #ifndef NDEBUG
1805 if( tmp_ps && std::find( tmp_procs, tmp_procs + tmp_ps, *sit ) != tmp_procs + tmp_ps )
1806 {
1807 std::cerr << "Trouble with something already in shared list on proc " << procConfig.proc_rank()
1808 << ". Entity:" << std::endl;
1809 list_entities( &entity, 1 );
1810 std::cerr << "pstatus = " << (int)pstatus << ", sharedp = " << sharedp << std::endl;
1811 std::cerr << "tmp_ps = ";
1812 for( int i = 0; i < tmp_ps; i++ )
1813 std::cerr << tmp_procs[i] << " ";
1814 std::cerr << std::endl;
1815 std::cerr << "procs = ";
1816 for( std::set< unsigned int >::iterator sit2 = procs.begin(); sit2 != procs.end(); ++sit2 )
1817 std::cerr << *sit2 << " ";
1818 assert( false );
1819 }
1820 #endif
1821 tmp_procs[num_ents] = *sit;
1822 tmp_handles[num_ents] = 0;
1823 num_ents++;
1824 }
1825
1826
1827 if( MAX_SHARING_PROCS > num_ents )
1828 {
1829 tmp_procs[num_ents] = -1;
1830 tmp_handles[num_ents] = 0;
1831 }
1832
1833 return MB_SUCCESS;
1834 }
1835
1836 ErrorCode ParallelComm::pack_entity_seq( const int nodes_per_entity,
1837 const bool store_remote_handles,
1838 const int to_proc,
1839 Range& these_ents,
1840 std::vector< EntityHandle >& entities_vec,
1841 Buffer* buff )
1842 {
1843 int tmp_space = 3 * sizeof( int ) + nodes_per_entity * these_ents.size() * sizeof( EntityHandle );
1844 buff->check_space( tmp_space );
1845
1846
1847 PACK_INT( buff->buff_ptr, ( (int)TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
1848
1849
1850 PACK_INT( buff->buff_ptr, these_ents.size() );
1851
1852
1853 PACK_INT( buff->buff_ptr, nodes_per_entity );
1854 myDebug->tprintf( 3, "after some pack int %d \n", buff->get_current_size() );
1855
1856
1857 std::vector< EntityHandle > connect;
1858 ErrorCode result = MB_SUCCESS;
1859 for( Range::const_iterator rit = these_ents.begin(); rit != these_ents.end(); ++rit )
1860 {
1861 connect.clear();
1862 result = mbImpl->get_connectivity( &( *rit ), 1, connect, false );MB_CHK_SET_ERR( result, "Failed to get connectivity" );
1863 assert( (int)connect.size() == nodes_per_entity );
1864 result =
1865 get_remote_handles( store_remote_handles, &connect[0], &connect[0], connect.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
1866 PACK_EH( buff->buff_ptr, &connect[0], connect.size() );
1867 }
1868
1869 myDebug->tprintf( 3, "Packed %lu ents of type %s\n", (unsigned long)these_ents.size(),
1870 CN::EntityTypeName( TYPE_FROM_HANDLE( *these_ents.begin() ) ) );
1871
1872 return result;
1873 }
1874
1875 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
1876 EntityHandle* from_vec,
1877 EntityHandle* to_vec_tmp,
1878 int num_ents,
1879 int to_proc,
1880 const std::vector< EntityHandle >& new_ents )
1881 {
1882
1883
1884
1885 if( 0 == num_ents ) return MB_SUCCESS;
1886
1887
1888 std::vector< EntityHandle > tmp_vector;
1889 EntityHandle* to_vec = to_vec_tmp;
1890 if( to_vec == from_vec )
1891 {
1892 tmp_vector.resize( num_ents );
1893 to_vec = &tmp_vector[0];
1894 }
1895
1896 if( !store_remote_handles )
1897 {
1898 int err;
1899
1900 for( int i = 0; i < num_ents; i++ )
1901 {
1902 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
1903 assert( new_ents[ind] == from_vec[i] );
1904 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
1905 assert( to_vec[i] != 0 && !err && -1 != ind );
1906 }
1907 }
1908 else
1909 {
1910 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
1911 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
1912
1913
1914 std::vector< int > sharing_procs( num_ents );
1915 result = mbImpl->tag_get_data( shh_tag, from_vec, num_ents, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
1916 result = mbImpl->tag_get_data( shp_tag, from_vec, num_ents, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
1917 for( int j = 0; j < num_ents; j++ )
1918 {
1919 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
1920 }
1921
1922 EntityHandle tmp_handles[MAX_SHARING_PROCS];
1923 int tmp_procs[MAX_SHARING_PROCS];
1924 int i;
1925
1926 for( i = 0; i < num_ents; i++ )
1927 {
1928 if( !to_vec[i] )
1929 {
1930 result = mbImpl->tag_get_data( shps_tag, from_vec + i, 1, tmp_procs );
1931 if( MB_SUCCESS == result )
1932 {
1933 for( int j = 0; j < MAX_SHARING_PROCS; j++ )
1934 {
1935 if( -1 == tmp_procs[j] )
1936 break;
1937 else if( tmp_procs[j] == to_proc )
1938 {
1939 result = mbImpl->tag_get_data( shhs_tag, from_vec + i, 1, tmp_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
1940 to_vec[i] = tmp_handles[j];
1941 assert( to_vec[i] );
1942 break;
1943 }
1944 }
1945 }
1946 if( !to_vec[i] )
1947 {
1948 int j = std::lower_bound( new_ents.begin(), new_ents.end(), from_vec[i] ) - new_ents.begin();
1949 if( (int)new_ents.size() == j )
1950 {
1951 std::cout << "Failed to find new entity in send list, proc " << procConfig.proc_rank()
1952 << std::endl;
1953 for( int k = 0; k <= num_ents; k++ )
1954 std::cout << k << ": " << from_vec[k] << " " << to_vec[k] << std::endl;
1955 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
1956 }
1957 int err;
1958 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
1959 if( err )
1960 {
1961 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
1962 }
1963 }
1964 }
1965 }
1966 }
1967
1968
1969 if( to_vec_tmp == from_vec ) memcpy( from_vec, to_vec, num_ents * sizeof( EntityHandle ) );
1970
1971 return MB_SUCCESS;
1972 }
1973
1974 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
1975 const Range& from_range,
1976 EntityHandle* to_vec,
1977 int to_proc,
1978 const std::vector< EntityHandle >& new_ents )
1979 {
1980
1981
1982
1983 if( from_range.empty() ) return MB_SUCCESS;
1984
1985 if( !store_remote_handles )
1986 {
1987 int err;
1988
1989 Range::iterator rit;
1990 unsigned int i;
1991 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
1992 {
1993 int ind = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
1994 assert( new_ents[ind] == *rit );
1995 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, ind, err );
1996 assert( to_vec[i] != 0 && !err && -1 != ind );
1997 }
1998 }
1999 else
2000 {
2001 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
2002 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
2003
2004
2005 std::vector< int > sharing_procs( from_range.size() );
2006 result = mbImpl->tag_get_data( shh_tag, from_range, to_vec );MB_CHK_SET_ERR( result, "Failed to get shared handle tag for remote_handles" );
2007 result = mbImpl->tag_get_data( shp_tag, from_range, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharing proc tag in remote_handles" );
2008 for( unsigned int j = 0; j < from_range.size(); j++ )
2009 {
2010 if( to_vec[j] && sharing_procs[j] != to_proc ) to_vec[j] = 0;
2011 }
2012
2013 EntityHandle tmp_handles[MAX_SHARING_PROCS];
2014 int tmp_procs[MAX_SHARING_PROCS];
2015
2016 Range::iterator rit;
2017 unsigned int i;
2018 for( rit = from_range.begin(), i = 0; rit != from_range.end(); ++rit, i++ )
2019 {
2020 if( !to_vec[i] )
2021 {
2022 result = mbImpl->tag_get_data( shhs_tag, &( *rit ), 1, tmp_handles );
2023 if( MB_SUCCESS == result )
2024 {
2025 result = mbImpl->tag_get_data( shps_tag, &( *rit ), 1, tmp_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
2026 for( int j = 0; j < MAX_SHARING_PROCS; j++ )
2027 if( tmp_procs[j] == to_proc )
2028 {
2029 to_vec[i] = tmp_handles[j];
2030 break;
2031 }
2032 }
2033
2034 if( !to_vec[i] )
2035 {
2036 int j = std::lower_bound( new_ents.begin(), new_ents.end(), *rit ) - new_ents.begin();
2037 if( (int)new_ents.size() == j )
2038 {
2039 MB_SET_ERR( MB_FAILURE, "Failed to find new entity in send list" );
2040 }
2041 int err;
2042 to_vec[i] = CREATE_HANDLE( MBMAXTYPE, j, err );
2043 if( err )
2044 {
2045 MB_SET_ERR( MB_FAILURE, "Failed to create handle in remote_handles" );
2046 }
2047 }
2048 }
2049 }
2050 }
2051
2052 return MB_SUCCESS;
2053 }
2054
2055 ErrorCode ParallelComm::get_remote_handles( const bool store_remote_handles,
2056 const Range& from_range,
2057 Range& to_range,
2058 int to_proc,
2059 const std::vector< EntityHandle >& new_ents )
2060 {
2061 std::vector< EntityHandle > to_vector( from_range.size() );
2062
2063 ErrorCode result = get_remote_handles( store_remote_handles, from_range, &to_vector[0], to_proc, new_ents );MB_CHK_SET_ERR( result, "Failed to get remote handles" );
2064 std::copy( to_vector.begin(), to_vector.end(), range_inserter( to_range ) );
2065 return result;
2066 }
2067
2068 ErrorCode ParallelComm::unpack_entities( unsigned char*& buff_ptr,
2069 const bool store_remote_handles,
2070 const int ,
2071 const bool is_iface,
2072 std::vector< std::vector< EntityHandle > >& L1hloc,
2073 std::vector< std::vector< EntityHandle > >& L1hrem,
2074 std::vector< std::vector< int > >& L1p,
2075 std::vector< EntityHandle >& L2hloc,
2076 std::vector< EntityHandle >& L2hrem,
2077 std::vector< unsigned int >& L2p,
2078 std::vector< EntityHandle >& new_ents,
2079 const bool created_iface )
2080 {
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108 ErrorCode result;
2109 bool done = false;
2110 ReadUtilIface* ru = NULL;
2111
2112 result = mbImpl->query_interface( ru );MB_CHK_SET_ERR( result, "Failed to get ReadUtilIface" );
2113
2114
2115 int num_ents = 0;
2116 unsigned char* buff_save = buff_ptr;
2117 int i, j;
2118
2119 if( store_remote_handles )
2120 {
2121 UNPACK_INT( buff_ptr, num_ents );
2122
2123 buff_save = buff_ptr;
2124
2125
2126 for( i = 0; i < num_ents; i++ )
2127 {
2128 UNPACK_INT( buff_ptr, j );
2129 if( j < 0 )
2130 {
2131 std::cout << "Should be non-negative # proc/handles.";
2132 return MB_FAILURE;
2133 }
2134
2135 buff_ptr += j * ( sizeof( int ) + sizeof( EntityHandle ) );
2136 }
2137 }
2138
2139 std::vector< EntityHandle > msg_ents;
2140
2141 while( !done )
2142 {
2143 EntityType this_type = MBMAXTYPE;
2144 UNPACK_TYPE( buff_ptr, this_type );
2145 assert( this_type != MBENTITYSET );
2146
2147
2148 if( MBMAXTYPE == this_type ) break;
2149
2150
2151 int num_ents2, verts_per_entity = 0;
2152 UNPACK_INT( buff_ptr, num_ents2 );
2153
2154
2155 if( MBVERTEX != this_type && num_ents2 )
2156 {
2157 UNPACK_INT( buff_ptr, verts_per_entity );
2158 }
2159
2160 std::vector< int > ps( MAX_SHARING_PROCS, -1 );
2161 std::vector< EntityHandle > hs( MAX_SHARING_PROCS, 0 );
2162 for( int e = 0; e < num_ents2; e++ )
2163 {
2164
2165 EntityHandle new_h = 0;
2166 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
2167 double coords[3];
2168 int num_ps = -1;
2169
2170
2171
2172
2173
2174 if( store_remote_handles )
2175 {
2176
2177 UNPACK_INT( buff_save, num_ps );
2178 if( 0 >= num_ps )
2179 {
2180 std::cout << "Shouldn't ever be fewer than 1 procs here." << std::endl;
2181 return MB_FAILURE;
2182 }
2183
2184 UNPACK_INTS( buff_save, &ps[0], num_ps );
2185 UNPACK_EH( buff_save, &hs[0], num_ps );
2186 }
2187
2188 if( MBVERTEX == this_type )
2189 {
2190 UNPACK_DBLS( buff_ptr, coords, 3 );
2191 }
2192 else
2193 {
2194 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
2195 UNPACK_EH( buff_ptr, connect, verts_per_entity );
2196
2197
2198 result = get_local_handles( connect, verts_per_entity, msg_ents );MB_CHK_SET_ERR( result, "Failed to get local handles" );
2199 }
2200
2201
2202
2203
2204
2205 if( store_remote_handles )
2206 {
2207 result = find_existing_entity( is_iface, ps[0], hs[0], num_ps, connect, verts_per_entity, this_type,
2208 L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
2209 }
2210
2211
2212
2213
2214 bool created_here = false;
2215 if( !new_h && !is_iface )
2216 {
2217 if( MBVERTEX == this_type )
2218 {
2219
2220 result = mbImpl->create_vertex( coords, new_h );MB_CHK_SET_ERR( result, "Failed to make new vertex" );
2221 }
2222 else
2223 {
2224
2225 result = mbImpl->create_element( this_type, connect, verts_per_entity, new_h );MB_CHK_SET_ERR( result, "Failed to make new element" );
2226
2227
2228 result = ru->update_adjacencies( new_h, 1, verts_per_entity, connect );MB_CHK_SET_ERR( result, "Failed to update adjacencies" );
2229 }
2230
2231
2232 assert( new_h );
2233
2234 created_here = true;
2235 }
2236
2237
2238
2239
2240
2241
2242
2243 if( !is_iface )
2244 {
2245 assert( new_h );
2246 msg_ents.push_back( new_h );
2247 }
2248
2249 if( created_here ) new_ents.push_back( new_h );
2250
2251 if( new_h && store_remote_handles )
2252 {
2253 unsigned char new_pstat = 0x0;
2254 if( is_iface )
2255 {
2256 new_pstat = PSTATUS_INTERFACE;
2257
2258 int idx = std::min_element( &ps[0], &ps[0] + num_ps ) - &ps[0];
2259 if( idx )
2260 {
2261 std::swap( ps[0], ps[idx] );
2262 std::swap( hs[0], hs[idx] );
2263 }
2264
2265
2266 if( ( num_ps > 1 && ps[0] != (int)rank() ) ) new_pstat |= PSTATUS_NOT_OWNED;
2267 }
2268 else if( created_here )
2269 {
2270 if( created_iface )
2271 new_pstat = PSTATUS_NOT_OWNED;
2272 else
2273 new_pstat = PSTATUS_GHOST | PSTATUS_NOT_OWNED;
2274 }
2275
2276
2277 result = update_remote_data( new_h, &ps[0], &hs[0], num_ps, new_pstat );MB_CHK_SET_ERR( result, "unpack_entities" );
2278
2279
2280 if( store_remote_handles && !is_iface && num_ps > 2 )
2281 {
2282 L2hrem.push_back( hs[0] );
2283 L2hloc.push_back( new_h );
2284 L2p.push_back( ps[0] );
2285 }
2286
2287
2288 if( !is_iface )
2289 {
2290 for( j = 0; j < num_ps; j++ )
2291 {
2292 if( ps[j] == (int)procConfig.proc_rank() ) continue;
2293 int idx = get_buffers( ps[j] );
2294 if( idx == (int)L1hloc.size() )
2295 {
2296 L1hloc.resize( idx + 1 );
2297 L1hrem.resize( idx + 1 );
2298 L1p.resize( idx + 1 );
2299 }
2300
2301
2302 std::vector< EntityHandle >::iterator vit =
2303 std::find( L1hloc[idx].begin(), L1hloc[idx].end(), new_h );
2304 if( vit != L1hloc[idx].end() )
2305 {
2306
2307
2308 if( L1p[idx][vit - L1hloc[idx].begin()] != -1 && hs[j] )
2309 {
2310 L1hrem[idx][vit - L1hloc[idx].begin()] = hs[j];
2311 L1p[idx][vit - L1hloc[idx].begin()] = -1;
2312 }
2313 else
2314 continue;
2315 }
2316 else
2317 {
2318 if( !hs[j] )
2319 {
2320 assert( -1 != ps[0] && num_ps > 2 );
2321 L1p[idx].push_back( ps[0] );
2322 L1hrem[idx].push_back( hs[0] );
2323 }
2324 else
2325 {
2326 assert(
2327 "either this remote handle isn't in the remote list, or "
2328 "it's for another proc" &&
2329 ( std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) == L1hrem[idx].end() ||
2330 L1p[idx][std::find( L1hrem[idx].begin(), L1hrem[idx].end(), hs[j] ) -
2331 L1hrem[idx].begin()] != -1 ) );
2332 L1p[idx].push_back( -1 );
2333 L1hrem[idx].push_back( hs[j] );
2334 }
2335 L1hloc[idx].push_back( new_h );
2336 }
2337 }
2338 }
2339
2340 assert( "Shouldn't be here for non-shared entities" && -1 != num_ps );
2341 std::fill( &ps[0], &ps[num_ps], -1 );
2342 std::fill( &hs[0], &hs[num_ps], 0 );
2343 }
2344 }
2345
2346 myDebug->tprintf( 4, "Unpacked %d ents of type %s", num_ents2, CN::EntityTypeName( this_type ) );
2347 }
2348
2349 myDebug->tprintf( 4, "Done unpacking entities.\n" );
2350
2351
2352 std::sort( new_ents.begin(), new_ents.end() );
2353
2354 return MB_SUCCESS;
2355 }
2356
2357 ErrorCode ParallelComm::print_buffer( unsigned char* buff_ptr, int mesg_tag, int from_proc, bool sent )
2358 {
2359 std::cerr << procConfig.proc_rank();
2360 if( sent )
2361 std::cerr << " sent";
2362 else
2363 std::cerr << " received";
2364 std::cerr << " message type " << mesg_tag << " to/from proc " << from_proc << "; contents:" << std::endl;
2365
2366 int msg_length, num_ents;
2367 unsigned char* orig_ptr = buff_ptr;
2368 UNPACK_INT( buff_ptr, msg_length );
2369 std::cerr << msg_length << " bytes..." << std::endl;
2370
2371 if( MB_MESG_ENTS_SIZE == mesg_tag || MB_MESG_ENTS_LARGE == mesg_tag )
2372 {
2373
2374 int i, j, k;
2375 std::vector< int > ps;
2376 std::vector< EntityHandle > hs;
2377
2378 UNPACK_INT( buff_ptr, num_ents );
2379 std::cerr << num_ents << " entities..." << std::endl;
2380
2381
2382 for( i = 0; i < num_ents; i++ )
2383 {
2384 UNPACK_INT( buff_ptr, j );
2385 if( 0 > j ) return MB_FAILURE;
2386 ps.resize( j );
2387 hs.resize( j );
2388 std::cerr << "Entity " << i << ", # procs = " << j << std::endl;
2389 UNPACK_INTS( buff_ptr, &ps[0], j );
2390 UNPACK_EH( buff_ptr, &hs[0], j );
2391 std::cerr << " Procs: ";
2392 for( k = 0; k < j; k++ )
2393 std::cerr << ps[k] << " ";
2394 std::cerr << std::endl;
2395 std::cerr << " Handles: ";
2396 for( k = 0; k < j; k++ )
2397 std::cerr << hs[k] << " ";
2398 std::cerr << std::endl;
2399
2400 if( buff_ptr - orig_ptr > msg_length )
2401 {
2402 std::cerr << "End of buffer..." << std::endl;
2403 std::cerr.flush();
2404 return MB_FAILURE;
2405 }
2406 }
2407
2408 while( true )
2409 {
2410 EntityType this_type = MBMAXTYPE;
2411 UNPACK_TYPE( buff_ptr, this_type );
2412 assert( this_type != MBENTITYSET );
2413
2414
2415 if( MBMAXTYPE == this_type ) break;
2416
2417
2418 int num_ents2, verts_per_entity = 0;
2419 UNPACK_INT( buff_ptr, num_ents2 );
2420
2421
2422 if( MBVERTEX != this_type && num_ents2 )
2423 {
2424 UNPACK_INT( buff_ptr, verts_per_entity );
2425 }
2426
2427 std::cerr << "Type: " << CN::EntityTypeName( this_type ) << "; num_ents = " << num_ents2;
2428 if( MBVERTEX != this_type ) std::cerr << "; verts_per_ent = " << verts_per_entity;
2429 std::cerr << std::endl;
2430 if( num_ents2 < 0 || num_ents2 > msg_length )
2431 {
2432 std::cerr << "Wrong number of entities, returning." << std::endl;
2433 return MB_FAILURE;
2434 }
2435
2436 for( int e = 0; e < num_ents2; e++ )
2437 {
2438
2439 if( MBVERTEX == this_type )
2440 {
2441 double coords[3];
2442 UNPACK_DBLS( buff_ptr, coords, 3 );
2443 std::cerr << "xyz = " << coords[0] << ", " << coords[1] << ", " << coords[2] << std::endl;
2444 }
2445 else
2446 {
2447 EntityHandle connect[CN::MAX_NODES_PER_ELEMENT];
2448 assert( verts_per_entity <= CN::MAX_NODES_PER_ELEMENT );
2449 UNPACK_EH( buff_ptr, connect, verts_per_entity );
2450
2451
2452 std::cerr << "Connectivity: ";
2453 for( k = 0; k < verts_per_entity; k++ )
2454 std::cerr << connect[k] << " ";
2455 std::cerr << std::endl;
2456 }
2457
2458 if( buff_ptr - orig_ptr > msg_length )
2459 {
2460 std::cerr << "End of buffer..." << std::endl;
2461 std::cerr.flush();
2462 return MB_FAILURE;
2463 }
2464 }
2465 }
2466 }
2467 else if( MB_MESG_REMOTEH_SIZE == mesg_tag || MB_MESG_REMOTEH_LARGE == mesg_tag )
2468 {
2469 UNPACK_INT( buff_ptr, num_ents );
2470 std::cerr << num_ents << " entities..." << std::endl;
2471 if( 0 > num_ents || num_ents > msg_length )
2472 {
2473 std::cerr << "Wrong number of entities, returning." << std::endl;
2474 return MB_FAILURE;
2475 }
2476 std::vector< EntityHandle > L1hloc( num_ents ), L1hrem( num_ents );
2477 std::vector< int > L1p( num_ents );
2478 UNPACK_INTS( buff_ptr, &L1p[0], num_ents );
2479 UNPACK_EH( buff_ptr, &L1hrem[0], num_ents );
2480 UNPACK_EH( buff_ptr, &L1hloc[0], num_ents );
2481 std::cerr << num_ents << " Entity pairs; hremote/hlocal/proc: " << std::endl;
2482 for( int i = 0; i < num_ents; i++ )
2483 {
2484 EntityType etype = TYPE_FROM_HANDLE( L1hloc[i] );
2485 std::cerr << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hrem[i] ) << ", "
2486 << CN::EntityTypeName( etype ) << ID_FROM_HANDLE( L1hloc[i] ) << ", " << L1p[i] << std::endl;
2487 }
2488
2489 if( buff_ptr - orig_ptr > msg_length )
2490 {
2491 std::cerr << "End of buffer..." << std::endl;
2492 std::cerr.flush();
2493 return MB_FAILURE;
2494 }
2495 }
2496 else if( mesg_tag == MB_MESG_TAGS_SIZE || mesg_tag == MB_MESG_TAGS_LARGE )
2497 {
2498 int num_tags, dum1, data_type, tag_size;
2499 UNPACK_INT( buff_ptr, num_tags );
2500 std::cerr << "Number of tags = " << num_tags << std::endl;
2501 for( int i = 0; i < num_tags; i++ )
2502 {
2503 std::cerr << "Tag " << i << ":" << std::endl;
2504 UNPACK_INT( buff_ptr, tag_size );
2505 UNPACK_INT( buff_ptr, dum1 );
2506 UNPACK_INT( buff_ptr, data_type );
2507 std::cerr << "Tag size, type, data type = " << tag_size << ", " << dum1 << ", " << data_type << std::endl;
2508 UNPACK_INT( buff_ptr, dum1 );
2509 std::cerr << "Default value size = " << dum1 << std::endl;
2510 buff_ptr += dum1;
2511 UNPACK_INT( buff_ptr, dum1 );
2512 std::string name( (char*)buff_ptr, dum1 );
2513 std::cerr << "Tag name = " << name.c_str() << std::endl;
2514 buff_ptr += dum1;
2515 UNPACK_INT( buff_ptr, num_ents );
2516 std::cerr << "Number of ents = " << num_ents << std::endl;
2517 std::vector< EntityHandle > tmp_buff( num_ents );
2518 UNPACK_EH( buff_ptr, &tmp_buff[0], num_ents );
2519 int tot_length = 0;
2520 for( int j = 0; j < num_ents; j++ )
2521 {
2522 EntityType etype = TYPE_FROM_HANDLE( tmp_buff[j] );
2523 std::cerr << CN::EntityTypeName( etype ) << " " << ID_FROM_HANDLE( tmp_buff[j] ) << ", tag = ";
2524 if( tag_size == MB_VARIABLE_LENGTH )
2525 {
2526 UNPACK_INT( buff_ptr, dum1 );
2527 tot_length += dum1;
2528 std::cerr << "(variable, length = " << dum1 << ")" << std::endl;
2529 }
2530 else if( data_type == MB_TYPE_DOUBLE )
2531 {
2532 double dum_dbl;
2533 UNPACK_DBL( buff_ptr, dum_dbl );
2534 std::cerr << dum_dbl << std::endl;
2535 }
2536 else if( data_type == MB_TYPE_INTEGER )
2537 {
2538 int dum_int;
2539 UNPACK_INT( buff_ptr, dum_int );
2540 std::cerr << dum_int << std::endl;
2541 }
2542 else if( data_type == MB_TYPE_OPAQUE )
2543 {
2544 std::cerr << "(opaque)" << std::endl;
2545 buff_ptr += tag_size;
2546 }
2547 else if( data_type == MB_TYPE_HANDLE )
2548 {
2549 EntityHandle dum_eh;
2550 UNPACK_EH( buff_ptr, &dum_eh, 1 );
2551 std::cerr << dum_eh << std::endl;
2552 }
2553 else if( data_type == MB_TYPE_BIT )
2554 {
2555 std::cerr << "(bit)" << std::endl;
2556 buff_ptr += tag_size;
2557 }
2558 }
2559 if( tag_size == MB_VARIABLE_LENGTH ) buff_ptr += tot_length;
2560 }
2561 }
2562 else
2563 {
2564 assert( false );
2565 return MB_FAILURE;
2566 }
2567
2568 std::cerr.flush();
2569
2570 return MB_SUCCESS;
2571 }
2572
2573 ErrorCode ParallelComm::list_entities( const EntityHandle* ents, int num_ents )
2574 {
2575 if( NULL == ents )
2576 {
2577 Range shared_ents;
2578 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) );
2579 shared_ents.print( "Shared entities:\n" );
2580 return MB_SUCCESS;
2581 }
2582
2583 unsigned char pstat;
2584 EntityHandle tmp_handles[MAX_SHARING_PROCS];
2585 int tmp_procs[MAX_SHARING_PROCS];
2586 unsigned int num_ps;
2587 ErrorCode result;
2588
2589 for( int i = 0; i < num_ents; i++ )
2590 {
2591 result = mbImpl->list_entities( ents + i, 1 );MB_CHK_ERR( result );
2592 double coords[3];
2593 result = mbImpl->get_coords( ents + i, 1, coords );
2594 std::cout << " coords: " << coords[0] << " " << coords[1] << " " << coords[2] << "\n";
2595
2596 result = get_sharing_data( ents[i], tmp_procs, tmp_handles, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
2597
2598 std::cout << "Pstatus: ";
2599 if( !num_ps )
2600 std::cout << "local " << std::endl;
2601 else
2602 {
2603 if( pstat & PSTATUS_NOT_OWNED ) std::cout << "NOT_OWNED; ";
2604 if( pstat & PSTATUS_SHARED ) std::cout << "SHARED; ";
2605 if( pstat & PSTATUS_MULTISHARED ) std::cout << "MULTISHARED; ";
2606 if( pstat & PSTATUS_INTERFACE ) std::cout << "INTERFACE; ";
2607 if( pstat & PSTATUS_GHOST ) std::cout << "GHOST; ";
2608 std::cout << std::endl;
2609 for( unsigned int j = 0; j < num_ps; j++ )
2610 {
2611 std::cout << " proc " << tmp_procs[j] << " id (handle) " << mbImpl->id_from_handle( tmp_handles[j] )
2612 << "(" << tmp_handles[j] << ")" << std::endl;
2613 }
2614 }
2615 std::cout << std::endl;
2616 }
2617
2618 return MB_SUCCESS;
2619 }
2620
2621 ErrorCode ParallelComm::list_entities( const Range& ents )
2622 {
2623 for( Range::iterator rit = ents.begin(); rit != ents.end(); ++rit )
2624 list_entities( &( *rit ), 1 );
2625
2626 return MB_SUCCESS;
2627 }
2628
2629 ErrorCode ParallelComm::update_remote_data( Range& local_range,
2630 Range& remote_range,
2631 int other_proc,
2632 const unsigned char add_pstat )
2633 {
2634 Range::iterator rit, rit2;
2635 ErrorCode result = MB_SUCCESS;
2636
2637
2638 for( rit = local_range.begin(), rit2 = remote_range.begin(); rit != local_range.end(); ++rit, ++rit2 )
2639 {
2640 result = update_remote_data( *rit, &other_proc, &( *rit2 ), 1, add_pstat );MB_CHK_ERR( result );
2641 }
2642
2643 return MB_SUCCESS;
2644 }
2645
2646 ErrorCode ParallelComm::update_remote_data( const EntityHandle new_h,
2647 const int* ps,
2648 const EntityHandle* hs,
2649 const int num_ps,
2650 const unsigned char add_pstat
2651
2652
2653
2654
2655
2656
2657 )
2658 {
2659
2660
2661
2662
2663
2664 int new_numps;
2665 unsigned char new_pstat;
2666 std::vector< int > new_ps( MAX_SHARING_PROCS, -1 );
2667 std::vector< EntityHandle > new_hs( MAX_SHARING_PROCS, 0 );
2668
2669 new_numps = 0;
2670 ErrorCode result = get_sharing_data( new_h, &new_ps[0], &new_hs[0], new_pstat, new_numps );MB_CHK_SET_ERR( result, "Failed to get sharing data in update_remote_data" );
2671 int num_exist = new_numps;
2672
2673
2674 new_pstat |= add_pstat;
2675
2676
2694
2695 #ifndef NDEBUG
2696 {
2697
2698 std::set< unsigned int > dumprocs;
2699 unsigned int dp = 0;
2700 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
2701 dumprocs.insert( ps[dp] );
2702 assert( dp == dumprocs.size() );
2703 }
2704 #endif
2705
2706
2707
2708 if( 1 == new_numps && !( new_pstat & PSTATUS_NOT_OWNED ) )
2709 {
2710 new_hs[1] = new_hs[0];
2711 new_ps[1] = new_ps[0];
2712 new_hs[0] = new_h;
2713 new_ps[0] = rank();
2714 new_numps = 2;
2715 }
2716
2717
2718 int idx;
2719 for( int i = 0; i < num_ps; i++ )
2720 {
2721 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, ps[i] ) - &new_ps[0];
2722 if( idx < new_numps )
2723 {
2724 if( !new_hs[idx] && hs[i] )
2725
2726 new_hs[idx] = hs[i];
2727 else
2728 assert( !hs[i] || new_hs[idx] == hs[i] );
2729 }
2730 else
2731 {
2732 if( new_numps + 1 == MAX_SHARING_PROCS )
2733 {
2734 MB_SET_ERR( MB_FAILURE, "Exceeded MAX_SHARING_PROCS for "
2735 << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) ) << ' '
2736 << ID_FROM_HANDLE( new_h ) << " in process " << rank() );
2737 }
2738 new_ps[new_numps] = ps[i];
2739 new_hs[new_numps] = hs[i];
2740 new_numps++;
2741 }
2742 }
2743
2744
2745 idx = std::find( &new_ps[0], &new_ps[0] + new_numps, rank() ) - &new_ps[0];
2746 if( idx == new_numps )
2747 {
2748 new_ps[new_numps] = rank();
2749 new_hs[new_numps] = new_h;
2750 new_numps++;
2751 }
2752 else if( !new_hs[idx] && new_numps > 2 )
2753 new_hs[idx] = new_h;
2754
2755
2756 if( new_numps > 1 )
2757 {
2758 if( new_numps > 2 ) new_pstat |= PSTATUS_MULTISHARED;
2759 new_pstat |= PSTATUS_SHARED;
2760 }
2761
2762
2769
2770 result = set_sharing_data( new_h, new_pstat, num_exist, new_numps, &new_ps[0], &new_hs[0] );MB_CHK_SET_ERR( result, "Failed to set sharing data in update_remote_data" );
2771
2772 if( new_pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
2773
2774 return MB_SUCCESS;
2775 }
2776
2777 ErrorCode ParallelComm::update_remote_data_old( const EntityHandle new_h,
2778 const int* ps,
2779 const EntityHandle* hs,
2780 const int num_ps,
2781 const unsigned char add_pstat )
2782 {
2783 EntityHandle tag_hs[MAX_SHARING_PROCS];
2784 int tag_ps[MAX_SHARING_PROCS];
2785 unsigned char pstat;
2786
2787
2788 unsigned int num_exist;
2789 ErrorCode result = get_sharing_data( new_h, tag_ps, tag_hs, pstat, num_exist );MB_CHK_ERR( result );
2790
2791 #ifndef NDEBUG
2792 {
2793
2794 std::set< unsigned int > dumprocs;
2795 unsigned int dp = 0;
2796 for( ; (int)dp < num_ps && -1 != ps[dp]; dp++ )
2797 dumprocs.insert( ps[dp] );
2798 assert( dp == dumprocs.size() );
2799 }
2800 #endif
2801
2802
2803 bool changed = false;
2804 int idx;
2805 if( !num_exist )
2806 {
2807
2808 memcpy( tag_ps, ps, num_ps * sizeof( int ) );
2809 memcpy( tag_hs, hs, num_ps * sizeof( EntityHandle ) );
2810 num_exist = num_ps;
2811
2812 assert( "I shouldn't be the only proc there." && ( 1 != num_exist || ps[0] != (int)procConfig.proc_rank() ) );
2813 changed = true;
2814 }
2815 else
2816 {
2817 for( int i = 0; i < num_ps; i++ )
2818 {
2819 idx = std::find( tag_ps, tag_ps + num_exist, ps[i] ) - tag_ps;
2820 if( idx == (int)num_exist )
2821 {
2822 if( num_exist == MAX_SHARING_PROCS )
2823 {
2824 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_h ) )
2825 << ' ' << ID_FROM_HANDLE( new_h ) << " in process " << proc_config().proc_rank()
2826 << std::endl;
2827 std::cerr.flush();
2828 MPI_Abort( proc_config().proc_comm(), 66 );
2829 }
2830
2831
2832
2833 if( !i && num_ps == 1 && num_exist == 1 && ps[0] != (int)procConfig.proc_rank() )
2834 {
2835 int j = 1;
2836
2837 if( !( pstat & PSTATUS_NOT_OWNED ) )
2838 {
2839 tag_ps[1] = tag_ps[0];
2840 tag_hs[1] = tag_hs[0];
2841 j = 0;
2842 }
2843 tag_ps[j] = procConfig.proc_rank();
2844 tag_hs[j] = new_h;
2845 num_exist++;
2846 }
2847
2848 tag_ps[num_exist] = ps[i];
2849 tag_hs[num_exist] = hs[i];
2850 num_exist++;
2851 changed = true;
2852 }
2853 else if( 0 == tag_hs[idx] )
2854 {
2855 tag_hs[idx] = hs[i];
2856 changed = true;
2857 }
2858 else if( 0 != hs[i] )
2859 {
2860 assert( hs[i] == tag_hs[idx] );
2861 }
2862 }
2863 }
2864
2865
2866 if( add_pstat & PSTATUS_INTERFACE )
2867 {
2868 idx = std::min_element( tag_ps, tag_ps + num_exist ) - tag_ps;
2869 if( idx )
2870 {
2871 int tag_proc = tag_ps[idx];
2872 tag_ps[idx] = tag_ps[0];
2873 tag_ps[0] = tag_proc;
2874 EntityHandle tag_h = tag_hs[idx];
2875 tag_hs[idx] = tag_hs[0];
2876 tag_hs[0] = tag_h;
2877 changed = true;
2878 if( tag_ps[0] != (int)procConfig.proc_rank() ) pstat |= PSTATUS_NOT_OWNED;
2879 }
2880 }
2881
2882 if( !changed ) return MB_SUCCESS;
2883
2884 assert( "interface entities should have > 1 proc" && ( !( add_pstat & PSTATUS_INTERFACE ) || num_exist > 1 ) );
2885 assert( "ghost entities should have > 1 proc" && ( !( add_pstat & PSTATUS_GHOST ) || num_exist > 1 ) );
2886
2887
2888
2889 if( num_exist > 2 )
2890 {
2891 idx = std::find( tag_ps, tag_ps + num_exist, procConfig.proc_rank() ) - tag_ps;
2892 assert( idx < (int)num_exist );
2893 if( !tag_hs[idx] ) tag_hs[idx] = new_h;
2894 }
2895
2896 int tag_p;
2897 EntityHandle tag_h;
2898
2899
2900 pstat |= add_pstat;
2901
2902 if( num_exist > 2 )
2903 pstat |= ( PSTATUS_MULTISHARED | PSTATUS_SHARED );
2904 else if( num_exist > 0 )
2905 pstat |= PSTATUS_SHARED;
2906
2907
2908
2909
2910
2911 if( num_exist > 2 && !( pstat & PSTATUS_MULTISHARED ) && ( pstat & PSTATUS_SHARED ) )
2912 {
2913
2914 tag_p = -1;
2915 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, &tag_p );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
2916 tag_h = 0;
2917 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, &tag_h );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
2918 }
2919
2920
2921 if( num_exist > 2 )
2922 {
2923 std::fill( tag_ps + num_exist, tag_ps + MAX_SHARING_PROCS, -1 );
2924 std::fill( tag_hs + num_exist, tag_hs + MAX_SHARING_PROCS, 0 );
2925 result = mbImpl->tag_set_data( sharedps_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" );
2926 result = mbImpl->tag_set_data( sharedhs_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" );
2927
2928 #ifndef NDEBUG
2929 {
2930
2931 std::set< unsigned int > dumprocs;
2932 unsigned int dp = 0;
2933 for( ; dp < num_exist && -1 != tag_ps[dp]; dp++ )
2934 dumprocs.insert( tag_ps[dp] );
2935 assert( dp == dumprocs.size() );
2936 }
2937 #endif
2938 }
2939 else if( num_exist == 2 || num_exist == 1 )
2940 {
2941 if( tag_ps[0] == (int)procConfig.proc_rank() )
2942 {
2943 assert( 2 == num_exist && tag_ps[1] != (int)procConfig.proc_rank() );
2944 tag_ps[0] = tag_ps[1];
2945 tag_hs[0] = tag_hs[1];
2946 }
2947 assert( tag_ps[0] != -1 && tag_hs[0] != 0 );
2948 result = mbImpl->tag_set_data( sharedp_tag(), &new_h, 1, tag_ps );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
2949 result = mbImpl->tag_set_data( sharedh_tag(), &new_h, 1, tag_hs );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
2950 }
2951
2952
2953 result = mbImpl->tag_set_data( pstatus_tag(), &new_h, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
2954
2955 if( pstat & PSTATUS_SHARED ) sharedEnts.insert( new_h );
2956
2957 return MB_SUCCESS;
2958 }
2959
2960 ErrorCode ParallelComm::get_sharing_data( const Range& entities, std::set< int >& procs, int operation )
2961 {
2962
2963 ErrorCode result;
2964 int sp2[MAX_SHARING_PROCS];
2965 int num_ps;
2966 unsigned char pstat;
2967 std::set< int > tmp_procs;
2968 procs.clear();
2969
2970 for( Range::const_iterator rit = entities.begin(); rit != entities.end(); ++rit )
2971 {
2972
2973 result = get_sharing_data( *rit, sp2, NULL, pstat, num_ps );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_sharing_data" );
2974 if( !( pstat & PSTATUS_SHARED ) && Interface::INTERSECT == operation )
2975 {
2976 procs.clear();
2977 return MB_SUCCESS;
2978 }
2979
2980 if( rit == entities.begin() )
2981 {
2982 std::copy( sp2, sp2 + num_ps, std::inserter( procs, procs.begin() ) );
2983 }
2984 else
2985 {
2986 std::sort( sp2, sp2 + num_ps );
2987 tmp_procs.clear();
2988 if( Interface::UNION == operation )
2989 std::set_union( procs.begin(), procs.end(), sp2, sp2 + num_ps,
2990 std::inserter( tmp_procs, tmp_procs.end() ) );
2991 else if( Interface::INTERSECT == operation )
2992 std::set_intersection( procs.begin(), procs.end(), sp2, sp2 + num_ps,
2993 std::inserter( tmp_procs, tmp_procs.end() ) );
2994 else
2995 {
2996 assert( "Unknown operation." && false );
2997 return MB_FAILURE;
2998 }
2999 procs.swap( tmp_procs );
3000 }
3001 if( Interface::INTERSECT == operation && procs.empty() ) return MB_SUCCESS;
3002 }
3003
3004 return MB_SUCCESS;
3005 }
3006
3007 ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity,
3008 int* ps,
3009 EntityHandle* hs,
3010 unsigned char& pstat,
3011 unsigned int& num_ps )
3012 {
3013 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
3014 if( pstat & PSTATUS_MULTISHARED )
3015 {
3016 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
3017 if( hs )
3018 {
3019 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
3020 }
3021 num_ps = std::find( ps, ps + MAX_SHARING_PROCS, -1 ) - ps;
3022 }
3023 else if( pstat & PSTATUS_SHARED )
3024 {
3025 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, ps );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
3026 if( hs )
3027 {
3028 result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, hs );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
3029 hs[1] = 0;
3030 }
3031
3032 ps[1] = -1;
3033 num_ps = 1;
3034 }
3035 else
3036 {
3037 ps[0] = -1;
3038 if( hs ) hs[0] = 0;
3039 num_ps = 0;
3040 }
3041
3042 assert( MAX_SHARING_PROCS >= num_ps );
3043
3044 return MB_SUCCESS;
3045 }
3046
3047 ErrorCode ParallelComm::find_existing_entity( const bool is_iface,
3048 const int owner_p,
3049 const EntityHandle owner_h,
3050 const int num_ps,
3051 const EntityHandle* connect,
3052 const int num_connect,
3053 const EntityType this_type,
3054 std::vector< EntityHandle >& L2hloc,
3055 std::vector< EntityHandle >& L2hrem,
3056 std::vector< unsigned int >& L2p,
3057 EntityHandle& new_h )
3058 {
3059 new_h = 0;
3060 if( !is_iface && num_ps > 2 )
3061 {
3062 for( unsigned int i = 0; i < L2hrem.size(); i++ )
3063 {
3064 if( L2hrem[i] == owner_h && owner_p == (int)L2p[i] )
3065 {
3066 new_h = L2hloc[i];
3067 return MB_SUCCESS;
3068 }
3069 }
3070 }
3071
3072
3073 if( MBVERTEX == this_type || !connect || !num_connect ) return MB_SUCCESS;
3074
3075 Range tmp_range;
3076 ErrorCode result = mbImpl->get_adjacencies( connect, num_connect, CN::Dimension( this_type ), false, tmp_range );MB_CHK_SET_ERR( result, "Failed to get existing entity" );
3077 if( !tmp_range.empty() )
3078 {
3079
3080 new_h = *tmp_range.begin();
3081 }
3082 else
3083 {
3084 new_h = 0;
3085 }
3086
3087 return MB_SUCCESS;
3088 }
3089
3090 ErrorCode ParallelComm::get_local_handles( const Range& remote_handles,
3091 Range& local_handles,
3092 const std::vector< EntityHandle >& new_ents )
3093 {
3094 std::vector< EntityHandle > rh_vec;
3095 rh_vec.reserve( remote_handles.size() );
3096 std::copy( remote_handles.begin(), remote_handles.end(), std::back_inserter( rh_vec ) );
3097 ErrorCode result = get_local_handles( &rh_vec[0], remote_handles.size(), new_ents );
3098 std::copy( rh_vec.begin(), rh_vec.end(), range_inserter( local_handles ) );
3099 return result;
3100 }
3101
3102 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents )
3103 {
3104 std::vector< EntityHandle > tmp_ents;
3105 std::copy( new_ents.begin(), new_ents.end(), std::back_inserter( tmp_ents ) );
3106 return get_local_handles( from_vec, num_ents, tmp_ents );
3107 }
3108
3109 ErrorCode ParallelComm::get_local_handles( EntityHandle* from_vec,
3110 int num_ents,
3111 const std::vector< EntityHandle >& new_ents )
3112 {
3113 for( int i = 0; i < num_ents; i++ )
3114 {
3115 if( TYPE_FROM_HANDLE( from_vec[i] ) == MBMAXTYPE )
3116 {
3117 assert( ID_FROM_HANDLE( from_vec[i] ) < (int)new_ents.size() );
3118 from_vec[i] = new_ents[ID_FROM_HANDLE( from_vec[i] )];
3119 }
3120 }
3121
3122 return MB_SUCCESS;
3123 }
3124
3125
3135
3136 ErrorCode ParallelComm::pack_range_map( Range& key_range, EntityHandle val_start, HandleMap& handle_map )
3137 {
3138 for( Range::const_pair_iterator key_it = key_range.const_pair_begin(); key_it != key_range.const_pair_end();
3139 ++key_it )
3140 {
3141 int tmp_num = ( *key_it ).second - ( *key_it ).first + 1;
3142 handle_map.insert( ( *key_it ).first, val_start, tmp_num );
3143 val_start += tmp_num;
3144 }
3145
3146 return MB_SUCCESS;
3147 }
3148
3149 ErrorCode ParallelComm::pack_sets( Range& entities, Buffer* buff, const bool store_remote_handles, const int to_proc )
3150 {
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165 ErrorCode result;
3166 Range all_sets = entities.subset_by_type( MBENTITYSET );
3167
3168 int buff_size = estimate_sets_buffer_size( all_sets, store_remote_handles );
3169 if( buff_size < 0 ) MB_SET_ERR( MB_FAILURE, "Failed to estimate sets buffer size" );
3170 buff->check_space( buff_size );
3171
3172
3173 PACK_INT( buff->buff_ptr, all_sets.size() );
3174
3175
3176 std::vector< unsigned int > options( all_sets.size() );
3177 Range::iterator rit;
3178 std::vector< EntityHandle > members;
3179 int i;
3180 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
3181 {
3182 result = mbImpl->get_meshset_options( *rit, options[i] );MB_CHK_SET_ERR( result, "Failed to get meshset options" );
3183 }
3184 buff->check_space( all_sets.size() * sizeof( unsigned int ) );
3185 PACK_VOID( buff->buff_ptr, &options[0], all_sets.size() * sizeof( unsigned int ) );
3186
3187
3188 if( !all_sets.empty() )
3189 {
3190 Tag uid_tag;
3191 int n_sets = all_sets.size();
3192 bool b_pack = false;
3193 std::vector< int > id_data( n_sets );
3194 result =
3195 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
3196
3197 result = mbImpl->tag_get_data( uid_tag, all_sets, &id_data[0] );
3198 if( MB_TAG_NOT_FOUND != result )
3199 {
3200 if( MB_SUCCESS != result ) MB_SET_ERR( result, "Failed to get parallel geometry unique ids" );
3201 for( i = 0; i < n_sets; i++ )
3202 {
3203 if( id_data[i] != 0 )
3204 {
3205 b_pack = true;
3206 break;
3207 }
3208 }
3209 }
3210
3211 if( b_pack )
3212 {
3213 buff->check_space( ( n_sets + 1 ) * sizeof( int ) );
3214 PACK_INT( buff->buff_ptr, n_sets );
3215 PACK_INTS( buff->buff_ptr, &id_data[0], n_sets );
3216 }
3217 else
3218 {
3219 buff->check_space( sizeof( int ) );
3220 PACK_INT( buff->buff_ptr, 0 );
3221 }
3222 }
3223
3224
3225 std::vector< EntityHandle > entities_vec( entities.size() );
3226 std::copy( entities.begin(), entities.end(), entities_vec.begin() );
3227 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
3228 {
3229 members.clear();
3230 result = mbImpl->get_entities_by_handle( *rit, members );MB_CHK_SET_ERR( result, "Failed to get entities in ordered set" );
3231 result =
3232 get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc, entities_vec );MB_CHK_SET_ERR( result, "Failed in get_remote_handles" );
3233 buff->check_space( members.size() * sizeof( EntityHandle ) + sizeof( int ) );
3234 PACK_INT( buff->buff_ptr, members.size() );
3235 PACK_EH( buff->buff_ptr, &members[0], members.size() );
3236 }
3237
3238
3239 if( !store_remote_handles )
3240 {
3241
3242 unsigned int tot_pch = 0;
3243 int num_pch;
3244 buff->check_space( 2 * all_sets.size() * sizeof( int ) );
3245 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
3246 {
3247
3248 result = mbImpl->num_parent_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num parents" );
3249 PACK_INT( buff->buff_ptr, num_pch );
3250 tot_pch += num_pch;
3251 result = mbImpl->num_child_meshsets( *rit, &num_pch );MB_CHK_SET_ERR( result, "Failed to get num children" );
3252 PACK_INT( buff->buff_ptr, num_pch );
3253 tot_pch += num_pch;
3254 }
3255
3256
3257 members.clear();
3258 members.reserve( tot_pch );
3259 std::vector< EntityHandle > tmp_pch;
3260 for( rit = all_sets.begin(), i = 0; rit != all_sets.end(); ++rit, i++ )
3261 {
3262 result = mbImpl->get_parent_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get parents" );
3263 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
3264 tmp_pch.clear();
3265 result = mbImpl->get_child_meshsets( *rit, tmp_pch );MB_CHK_SET_ERR( result, "Failed to get children" );
3266 std::copy( tmp_pch.begin(), tmp_pch.end(), std::back_inserter( members ) );
3267 tmp_pch.clear();
3268 }
3269 assert( members.size() == tot_pch );
3270 if( !members.empty() )
3271 {
3272 result = get_remote_handles( store_remote_handles, &members[0], &members[0], members.size(), to_proc,
3273 entities_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles for set parent/child sets" );
3274 #ifndef NDEBUG
3275
3276 for( unsigned int __j = 0; __j < members.size(); __j++ )
3277 assert( ( TYPE_FROM_HANDLE( members[__j] ) == MBMAXTYPE &&
3278 ID_FROM_HANDLE( members[__j] ) < (int)entities.size() ) ||
3279 TYPE_FROM_HANDLE( members[__j] ) == MBENTITYSET );
3280 #endif
3281 buff->check_space( members.size() * sizeof( EntityHandle ) );
3282 PACK_EH( buff->buff_ptr, &members[0], members.size() );
3283 }
3284 }
3285 else
3286 {
3287 buff->check_space( 2 * all_sets.size() * sizeof( int ) );
3288 for( rit = all_sets.begin(); rit != all_sets.end(); ++rit )
3289 {
3290 PACK_INT( buff->buff_ptr, 0 );
3291 PACK_INT( buff->buff_ptr, 0 );
3292 }
3293 }
3294
3295
3296 if( store_remote_handles && !all_sets.empty() )
3297 {
3298 buff_size = RANGE_SIZE( all_sets );
3299 buff->check_space( buff_size );
3300 PACK_RANGE( buff->buff_ptr, all_sets );
3301 }
3302
3303 myDebug->tprintf( 4, "Done packing sets.\n" );
3304
3305 buff->set_stored_size();
3306
3307 return MB_SUCCESS;
3308 }
3309
3310 ErrorCode ParallelComm::unpack_sets( unsigned char*& buff_ptr,
3311 std::vector< EntityHandle >& entities,
3312 const bool store_remote_handles,
3313 const int from_proc )
3314 {
3315
3316 ErrorCode result;
3317
3318 bool no_sets = ( entities.empty() || ( mbImpl->type_from_handle( *entities.rbegin() ) == MBENTITYSET ) );
3319
3320 Range new_sets;
3321 int num_sets;
3322 UNPACK_INT( buff_ptr, num_sets );
3323
3324 if( !num_sets ) return MB_SUCCESS;
3325
3326 int i;
3327 Range::const_iterator rit;
3328 std::vector< EntityHandle > members;
3329 int num_ents;
3330 std::vector< unsigned int > options_vec( num_sets );
3331
3332 if( num_sets ) UNPACK_VOID( buff_ptr, &options_vec[0], num_sets * sizeof( unsigned int ) );
3333
3334
3335 int n_uid;
3336 UNPACK_INT( buff_ptr, n_uid );
3337 if( n_uid > 0 && n_uid != num_sets )
3338 {
3339 std::cerr << "The number of Parallel geometry unique ids should be same." << std::endl;
3340 }
3341
3342 if( n_uid > 0 )
3343 {
3344 std::vector< int > uids( n_uid );
3345 UNPACK_INTS( buff_ptr, &uids[0], n_uid );
3346
3347 Tag uid_tag;
3348 result =
3349 mbImpl->tag_get_handle( "PARALLEL_UNIQUE_ID", 1, MB_TYPE_INTEGER, uid_tag, MB_TAG_SPARSE | MB_TAG_CREAT );MB_CHK_SET_ERR( result, "Failed to create parallel geometry unique id tag" );
3350
3351
3352 for( i = 0; i < n_uid; i++ )
3353 {
3354 EntityHandle set_handle;
3355 Range temp_sets;
3356 void* tag_vals[] = { &uids[i] };
3357 if( uids[i] > 0 )
3358 {
3359 result = mbImpl->get_entities_by_type_and_tag( 0, MBENTITYSET, &uid_tag, tag_vals, 1, temp_sets );
3360 }
3361 if( !temp_sets.empty() )
3362 {
3363 set_handle = *temp_sets.begin();
3364 }
3365 else
3366 {
3367 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
3368 result = mbImpl->tag_set_data( uid_tag, &set_handle, 1, &uids[i] );MB_CHK_SET_ERR( result, "Failed to set parallel geometry unique ids" );
3369 }
3370 new_sets.insert( set_handle );
3371 }
3372 }
3373 else
3374 {
3375
3376 for( i = 0; i < num_sets; i++ )
3377 {
3378 EntityHandle set_handle;
3379 result = mbImpl->create_meshset( options_vec[i], set_handle );MB_CHK_SET_ERR( result, "Failed to create set in unpack" );
3380
3381
3382 assert( set_handle > *new_sets.rbegin() );
3383 new_sets.insert( set_handle );
3384 }
3385 }
3386
3387 std::copy( new_sets.begin(), new_sets.end(), std::back_inserter( entities ) );
3388
3389 if( !no_sets ) std::sort( entities.begin(), entities.end() );
3390
3391 for( rit = new_sets.begin(), i = 0; rit != new_sets.end(); ++rit, i++ )
3392 {
3393
3394 UNPACK_INT( buff_ptr, num_ents );
3395 members.resize( num_ents );
3396 if( num_ents ) UNPACK_EH( buff_ptr, &members[0], num_ents );
3397 result = get_local_handles( &members[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for ordered set contents" );
3398 result = mbImpl->add_entities( *rit, &members[0], num_ents );MB_CHK_SET_ERR( result, "Failed to add ents to ordered set in unpack" );
3399 }
3400
3401 std::vector< int > num_pch( 2 * new_sets.size() );
3402 std::vector< int >::iterator vit;
3403 int tot_pch = 0;
3404 for( vit = num_pch.begin(); vit != num_pch.end(); ++vit )
3405 {
3406 UNPACK_INT( buff_ptr, *vit );
3407 tot_pch += *vit;
3408 }
3409
3410 members.resize( tot_pch );
3411 UNPACK_EH( buff_ptr, &members[0], tot_pch );
3412 result = get_local_handles( &members[0], tot_pch, entities );MB_CHK_SET_ERR( result, "Failed to get local handle for parent/child sets" );
3413
3414 int num = 0;
3415 EntityHandle* mem_ptr = &members[0];
3416 for( rit = new_sets.begin(); rit != new_sets.end(); ++rit )
3417 {
3418
3419 int num_par = num_pch[num++], num_child = num_pch[num++];
3420 if( num_par + num_child )
3421 {
3422 for( i = 0; i < num_par; i++ )
3423 {
3424 assert( 0 != mem_ptr[i] );
3425 result = mbImpl->add_parent_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add parent to set in unpack" );
3426 }
3427 mem_ptr += num_par;
3428 for( i = 0; i < num_child; i++ )
3429 {
3430 assert( 0 != mem_ptr[i] );
3431 result = mbImpl->add_child_meshset( *rit, mem_ptr[i] );MB_CHK_SET_ERR( result, "Failed to add child to set in unpack" );
3432 }
3433 mem_ptr += num_child;
3434 }
3435 }
3436
3437
3438 Range dum_range;
3439 if( store_remote_handles && !new_sets.empty() )
3440 {
3441 UNPACK_RANGE( buff_ptr, dum_range );
3442 result = update_remote_data( new_sets, dum_range, from_proc, 0 );MB_CHK_SET_ERR( result, "Failed to set sharing data for sets" );
3443 }
3444
3445 myDebug->tprintf( 4, "Done unpacking sets." );
3446
3447 return MB_SUCCESS;
3448 }
3449
3450 ErrorCode ParallelComm::pack_adjacencies( Range& ,
3451 Range::const_iterator& ,
3452 Range& ,
3453 unsigned char*& ,
3454 int& ,
3455 const bool ,
3456 const bool ,
3457 const int )
3458 {
3459 return MB_FAILURE;
3460 }
3461
3462 ErrorCode ParallelComm::unpack_adjacencies( unsigned char*& ,
3463 Range& ,
3464 const bool ,
3465 const int )
3466 {
3467 return MB_FAILURE;
3468 }
3469
3470 ErrorCode ParallelComm::pack_tags( Range& entities,
3471 const std::vector< Tag >& src_tags,
3472 const std::vector< Tag >& dst_tags,
3473 const std::vector< Range >& tag_ranges,
3474 Buffer* buff,
3475 const bool store_remote_handles,
3476 const int to_proc )
3477 {
3478 ErrorCode result;
3479 std::vector< Tag >::const_iterator tag_it, dst_it;
3480 std::vector< Range >::const_iterator rit;
3481 int count = 0;
3482
3483 for( tag_it = src_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end(); ++tag_it, ++rit )
3484 {
3485 result = packed_tag_size( *tag_it, *rit, count );
3486 if( MB_SUCCESS != result ) return result;
3487 }
3488
3489
3490 count += sizeof( int );
3491
3492 buff->check_space( count );
3493
3494 PACK_INT( buff->buff_ptr, src_tags.size() );
3495
3496 std::vector< EntityHandle > entities_vec( entities.size() );
3497 std::copy( entities.begin(), entities.end(), entities_vec.begin() );
3498
3499 for( tag_it = src_tags.begin(), dst_it = dst_tags.begin(), rit = tag_ranges.begin(); tag_it != src_tags.end();
3500 ++tag_it, ++dst_it, ++rit )
3501 {
3502 result = pack_tag( *tag_it, *dst_it, *rit, entities_vec, buff, store_remote_handles, to_proc );
3503 if( MB_SUCCESS != result ) return result;
3504 }
3505
3506 myDebug->tprintf( 4, "Done packing tags." );
3507
3508 buff->set_stored_size();
3509
3510 return MB_SUCCESS;
3511 }
3512
3513 ErrorCode ParallelComm::packed_tag_size( Tag tag, const Range& tagged_entities, int& count )
3514 {
3515
3516
3517
3518 std::vector< int > var_len_sizes;
3519 std::vector< const void* > var_len_values;
3520
3521
3522 count += sizeof( int );
3523 if( NULL != tag->get_default_value() ) count += tag->get_default_value_size();
3524
3525
3526 count += 3 * sizeof( int );
3527
3528
3529 count += sizeof( int );
3530 count += tag->get_name().size();
3531
3532
3533 count += sizeof( int ) + tagged_entities.size() * sizeof( EntityHandle );
3534
3535 if( tag->get_size() == MB_VARIABLE_LENGTH )
3536 {
3537 const int num_ent = tagged_entities.size();
3538
3539 count += num_ent * sizeof( int );
3540
3541 var_len_sizes.resize( num_ent );
3542 var_len_values.resize( num_ent );
3543 ErrorCode result =
3544 tag->get_data( sequenceManager, errorHandler, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get lenghts of variable-length tag values" );
3545 count += std::accumulate( var_len_sizes.begin(), var_len_sizes.end(), 0 );
3546 }
3547 else
3548 {
3549
3550 count += tagged_entities.size() * tag->get_size();
3551 }
3552
3553 return MB_SUCCESS;
3554 }
3555
3556 ErrorCode ParallelComm::pack_tag( Tag src_tag,
3557 Tag dst_tag,
3558 const Range& tagged_entities,
3559 const std::vector< EntityHandle >& whole_vec,
3560 Buffer* buff,
3561 const bool store_remote_handles,
3562 const int to_proc )
3563 {
3564 ErrorCode result;
3565 std::vector< int > var_len_sizes;
3566 std::vector< const void* > var_len_values;
3567
3568 if( src_tag != dst_tag )
3569 {
3570 if( dst_tag->get_size() != src_tag->get_size() ) return MB_TYPE_OUT_OF_RANGE;
3571 if( dst_tag->get_data_type() != src_tag->get_data_type() && dst_tag->get_data_type() != MB_TYPE_OPAQUE &&
3572 src_tag->get_data_type() != MB_TYPE_OPAQUE )
3573 return MB_TYPE_OUT_OF_RANGE;
3574 }
3575
3576
3577 buff->check_space( 3 * sizeof( int ) );
3578 PACK_INT( buff->buff_ptr, src_tag->get_size() );
3579 TagType this_type;
3580 result = mbImpl->tag_get_type( dst_tag, this_type );
3581 PACK_INT( buff->buff_ptr, (int)this_type );
3582 DataType data_type = src_tag->get_data_type();
3583 PACK_INT( buff->buff_ptr, (int)data_type );
3584 int type_size = TagInfo::size_from_data_type( data_type );
3585
3586
3587 if( NULL == src_tag->get_default_value() )
3588 {
3589 buff->check_space( sizeof( int ) );
3590 PACK_INT( buff->buff_ptr, 0 );
3591 }
3592 else
3593 {
3594 buff->check_space( src_tag->get_default_value_size() );
3595 PACK_BYTES( buff->buff_ptr, src_tag->get_default_value(), src_tag->get_default_value_size() );
3596 }
3597
3598
3599 buff->check_space( src_tag->get_name().size() );
3600 PACK_BYTES( buff->buff_ptr, dst_tag->get_name().c_str(), dst_tag->get_name().size() );
3601
3602 myDebug->tprintf( 4, "Packing tag \"%s\"", src_tag->get_name().c_str() );
3603 if( src_tag != dst_tag ) myDebug->tprintf( 4, " (as tag \"%s\")", dst_tag->get_name().c_str() );
3604 myDebug->tprintf( 4, "\n" );
3605
3606
3607 buff->check_space( tagged_entities.size() * sizeof( EntityHandle ) + sizeof( int ) );
3608 PACK_INT( buff->buff_ptr, tagged_entities.size() );
3609 std::vector< EntityHandle > dum_tagged_entities( tagged_entities.size() );
3610 result = get_remote_handles( store_remote_handles, tagged_entities, &dum_tagged_entities[0], to_proc, whole_vec );
3611 if( MB_SUCCESS != result )
3612 {
3613 if( myDebug->get_verbosity() == 3 )
3614 {
3615 std::cerr << "Failed to get remote handles for tagged entities:" << std::endl;
3616 tagged_entities.print( " " );
3617 }
3618 MB_SET_ERR( result, "Failed to get remote handles for tagged entities" );
3619 }
3620
3621 PACK_EH( buff->buff_ptr, &dum_tagged_entities[0], dum_tagged_entities.size() );
3622
3623 const size_t num_ent = tagged_entities.size();
3624 if( src_tag->get_size() == MB_VARIABLE_LENGTH )
3625 {
3626 var_len_sizes.resize( num_ent, 0 );
3627 var_len_values.resize( num_ent, 0 );
3628 result = mbImpl->tag_get_by_ptr( src_tag, tagged_entities, &var_len_values[0], &var_len_sizes[0] );MB_CHK_SET_ERR( result, "Failed to get variable-length tag data in pack_tags" );
3629 buff->check_space( num_ent * sizeof( int ) );
3630 PACK_INTS( buff->buff_ptr, &var_len_sizes[0], num_ent );
3631 for( unsigned int i = 0; i < num_ent; i++ )
3632 {
3633 buff->check_space( var_len_sizes[i] );
3634 PACK_VOID( buff->buff_ptr, var_len_values[i], type_size * var_len_sizes[i] );
3635 }
3636 }
3637 else
3638 {
3639 buff->check_space( num_ent * src_tag->get_size() );
3640
3641
3642 result = mbImpl->tag_get_data( src_tag, tagged_entities, buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get tag data in pack_tags" );
3643 buff->buff_ptr += num_ent * src_tag->get_size();
3644 PC( num_ent * src_tag->get_size(), " void" );
3645 }
3646
3647 return MB_SUCCESS;
3648 }
3649
3650 ErrorCode ParallelComm::get_tag_send_list( const Range& whole_range,
3651 std::vector< Tag >& all_tags,
3652 std::vector< Range >& tag_ranges )
3653 {
3654 std::vector< Tag > tmp_tags;
3655 ErrorCode result = mbImpl->tag_get_tags( tmp_tags );MB_CHK_SET_ERR( result, "Failed to get tags in pack_tags" );
3656
3657 std::vector< Tag >::iterator tag_it;
3658 for( tag_it = tmp_tags.begin(); tag_it != tmp_tags.end(); ++tag_it )
3659 {
3660 std::string tag_name;
3661 result = mbImpl->tag_get_name( *tag_it, tag_name );
3662 if( tag_name.c_str()[0] == '_' && tag_name.c_str()[1] == '_' ) continue;
3663
3664 Range tmp_range;
3665 result = ( *tag_it )->get_tagged_entities( sequenceManager, tmp_range );MB_CHK_SET_ERR( result, "Failed to get entities for tag in pack_tags" );
3666 tmp_range = intersect( tmp_range, whole_range );
3667
3668 if( tmp_range.empty() ) continue;
3669
3670
3671 all_tags.push_back( *tag_it );
3672 tag_ranges.push_back( Range() );
3673 tag_ranges.back().swap( tmp_range );
3674 }
3675
3676 return MB_SUCCESS;
3677 }
3678
3679 ErrorCode ParallelComm::unpack_tags( unsigned char*& buff_ptr,
3680 std::vector< EntityHandle >& entities,
3681 const bool ,
3682 const int ,
3683 const MPI_Op* const mpi_op )
3684 {
3685
3686
3687
3688
3689
3690 ErrorCode result;
3691
3692 int num_tags;
3693 UNPACK_INT( buff_ptr, num_tags );
3694 std::vector< const void* > var_len_vals;
3695 std::vector< unsigned char > dum_vals;
3696 std::vector< EntityHandle > dum_ehvals;
3697
3698 for( int i = 0; i < num_tags; i++ )
3699 {
3700
3701 Tag tag_handle;
3702
3703
3704 int tag_size, tag_data_type, tag_type;
3705 UNPACK_INT( buff_ptr, tag_size );
3706 UNPACK_INT( buff_ptr, tag_type );
3707 UNPACK_INT( buff_ptr, tag_data_type );
3708
3709
3710 int def_val_size;
3711 UNPACK_INT( buff_ptr, def_val_size );
3712 void* def_val_ptr = NULL;
3713 if( def_val_size )
3714 {
3715 def_val_ptr = buff_ptr;
3716 buff_ptr += def_val_size;
3717 UPC( tag_size, " void" );
3718 }
3719
3720
3721 int name_len;
3722 UNPACK_INT( buff_ptr, name_len );
3723 std::string tag_name( reinterpret_cast< char* >( buff_ptr ), name_len );
3724 buff_ptr += name_len;
3725 UPC( 64, " chars" );
3726
3727 myDebug->tprintf( 4, "Unpacking tag %s\n", tag_name.c_str() );
3728
3729
3730 if( tag_size == MB_VARIABLE_LENGTH )
3731 result = mbImpl->tag_get_handle( tag_name.c_str(), def_val_size, (DataType)tag_data_type, tag_handle,
3732 MB_TAG_VARLEN | MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
3733 else
3734 result = mbImpl->tag_get_handle( tag_name.c_str(), tag_size, (DataType)tag_data_type, tag_handle,
3735 MB_TAG_CREAT | MB_TAG_BYTES | tag_type, def_val_ptr );
3736 if( MB_SUCCESS != result ) return result;
3737
3738
3739 int num_ents;
3740 UNPACK_INT( buff_ptr, num_ents );
3741 std::vector< EntityHandle > dum_ents( num_ents );
3742 UNPACK_EH( buff_ptr, &dum_ents[0], num_ents );
3743
3744
3745
3746 result = get_local_handles( &dum_ents[0], num_ents, entities );MB_CHK_SET_ERR( result, "Unable to convert to local handles" );
3747
3748
3749 if( MB_TYPE_HANDLE == tag_type )
3750 {
3751 dum_ehvals.resize( num_ents );
3752 UNPACK_EH( buff_ptr, &dum_ehvals[0], num_ents );
3753 result = get_local_handles( &dum_ehvals[0], num_ents, entities );MB_CHK_SET_ERR( result, "Failed to get local handles for tag vals" );
3754 }
3755
3756 DataType data_type;
3757 mbImpl->tag_get_data_type( tag_handle, data_type );
3758 int type_size = TagInfo::size_from_data_type( data_type );
3759
3760 if( !dum_ents.empty() )
3761 {
3762 if( tag_size == MB_VARIABLE_LENGTH )
3763 {
3764
3765
3766
3767 std::vector< int > var_lengths( num_ents );
3768 UNPACK_INTS( buff_ptr, &var_lengths[0], num_ents );
3769 UPC( sizeof( int ) * num_ents, " void" );
3770
3771
3772 var_len_vals.resize( num_ents );
3773 for( std::vector< EntityHandle >::size_type j = 0; j < (std::vector< EntityHandle >::size_type)num_ents;
3774 j++ )
3775 {
3776 var_len_vals[j] = buff_ptr;
3777 buff_ptr += var_lengths[j] * type_size;
3778 UPC( var_lengths[j], " void" );
3779 }
3780 result =
3781 mbImpl->tag_set_by_ptr( tag_handle, &dum_ents[0], num_ents, &var_len_vals[0], &var_lengths[0] );MB_CHK_SET_ERR( result, "Failed to set tag data when unpacking variable-length tag" );
3782 }
3783 else
3784 {
3785
3786 dum_vals.resize( tag_size * num_ents );
3787 if( mpi_op )
3788 {
3789 int tag_length;
3790 result = mbImpl->tag_get_length( tag_handle, tag_length );MB_CHK_SET_ERR( result, "Failed to get tag length" );
3791 result = mbImpl->tag_get_data( tag_handle, &dum_ents[0], num_ents, &dum_vals[0] );MB_CHK_SET_ERR( result, "Failed to get existing value of dst tag on entities" );
3792 result = reduce_void( tag_data_type, *mpi_op, tag_length * num_ents, &dum_vals[0], buff_ptr );MB_CHK_SET_ERR( result, "Failed to perform mpi op on dst tags" );
3793 }
3794 result = mbImpl->tag_set_data( tag_handle, &dum_ents[0], num_ents, buff_ptr );MB_CHK_SET_ERR( result, "Failed to set range-based tag data when unpacking tag" );
3795 buff_ptr += num_ents * tag_size;
3796 UPC( num_ents * tag_size, " void" );
3797 }
3798 }
3799 }
3800
3801 myDebug->tprintf( 4, "Done unpacking tags.\n" );
3802
3803 return MB_SUCCESS;
3804 }
3805
3806 template < class T >
3807 T LAND( const T& arg1, const T& arg2 )
3808 {
3809 return arg1 && arg2;
3810 }
3811 template < class T >
3812 T LOR( const T& arg1, const T& arg2 )
3813 {
3814 return arg1 || arg2;
3815 }
3816 template < class T >
3817 T LXOR( const T& arg1, const T& arg2 )
3818 {
3819 return ( ( arg1 && !arg2 ) || ( !arg1 && arg2 ) );
3820 }
3821 template < class T >
3822 T MAX( const T& arg1, const T& arg2 )
3823 {
3824 return ( arg1 > arg2 ? arg1 : arg2 );
3825 }
3826 template < class T >
3827 T MIN( const T& arg1, const T& arg2 )
3828 {
3829 return ( arg1 < arg2 ? arg1 : arg2 );
3830 }
3831 template < class T >
3832 T ADD( const T& arg1, const T& arg2 )
3833 {
3834 return arg1 + arg2;
3835 }
3836 template < class T >
3837 T MULT( const T& arg1, const T& arg2 )
3838 {
3839 return arg1 * arg2;
3840 }
3841
3842 template < class T >
3843 ErrorCode ParallelComm::reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals )
3844 {
3845 T* old_tmp = reinterpret_cast< T* >( old_vals );
3846
3847
3848
3849 std::vector< T > new_values;
3850 new_values.resize( num_ents );
3851 memcpy( &new_values[0], new_vals, num_ents * sizeof( T ) );
3852 T* new_tmp = &new_values[0];
3853
3854 if( mpi_op == MPI_SUM )
3855 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, ADD< T > );
3856 else if( mpi_op == MPI_PROD )
3857 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MULT< T > );
3858 else if( mpi_op == MPI_MAX )
3859 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MAX< T > );
3860 else if( mpi_op == MPI_MIN )
3861 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, MIN< T > );
3862 else if( mpi_op == MPI_LAND )
3863 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LAND< T > );
3864 else if( mpi_op == MPI_LOR )
3865 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LOR< T > );
3866 else if( mpi_op == MPI_LXOR )
3867 std::transform( old_tmp, old_tmp + num_ents, new_tmp, new_tmp, LXOR< T > );
3868 else if( mpi_op == MPI_BAND || mpi_op == MPI_BOR || mpi_op == MPI_BXOR )
3869 {
3870 std::cerr << "Bitwise operations not allowed in tag reductions." << std::endl;
3871 return MB_FAILURE;
3872 }
3873 else if( mpi_op != MPI_OP_NULL )
3874 {
3875 std::cerr << "Unknown MPI operation type." << std::endl;
3876 return MB_TYPE_OUT_OF_RANGE;
3877 }
3878
3879
3880 memcpy( new_vals, new_tmp, num_ents * sizeof( T ) );
3881 std::vector< T >().swap( new_values );
3882
3883 return MB_SUCCESS;
3884 }
3885
3886 ErrorCode ParallelComm::reduce_void( int tag_data_type,
3887 const MPI_Op mpi_op,
3888 int num_ents,
3889 void* old_vals,
3890 void* new_vals )
3891 {
3892 ErrorCode result;
3893 switch( tag_data_type )
3894 {
3895 case MB_TYPE_INTEGER:
3896 result = reduce< int >( mpi_op, num_ents, old_vals, new_vals );
3897 break;
3898 case MB_TYPE_DOUBLE:
3899 result = reduce< double >( mpi_op, num_ents, old_vals, new_vals );
3900 break;
3901 case MB_TYPE_BIT:
3902 result = reduce< unsigned char >( mpi_op, num_ents, old_vals, new_vals );
3903 break;
3904 default:
3905 result = MB_SUCCESS;
3906 break;
3907 }
3908
3909 return result;
3910 }
3911
3912 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set, int resolve_dim, int shared_dim, const Tag* id_tag )
3913 {
3914 ErrorCode result;
3915 Range proc_ents;
3916
3917
3918 ScdInterface* scdi;
3919 result = mbImpl->query_interface( scdi );
3920 if( scdi )
3921 {
3922 result = scdi->tag_shared_vertices( this, this_set );
3923 if( MB_SUCCESS == result )
3924 {
3925 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
3926 return result;
3927 }
3928 }
3929
3930 if( 0 == this_set )
3931 {
3932
3933 for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit )
3934 {
3935 Range tmp_ents;
3936 result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true );
3937 if( MB_SUCCESS != result ) return result;
3938 proc_ents.merge( tmp_ents );
3939 }
3940 }
3941 else
3942 {
3943 result = mbImpl->get_entities_by_handle( this_set, proc_ents, true );
3944 if( MB_SUCCESS != result ) return result;
3945 }
3946
3947
3948 if( -1 == resolve_dim )
3949 {
3950 if( !proc_ents.empty() ) resolve_dim = mbImpl->dimension_from_handle( *proc_ents.rbegin() );
3951 }
3952
3953
3954 if( resolve_dim > shared_dim &&
3955 mbImpl->dimension_from_handle( *proc_ents.rbegin() ) != mbImpl->dimension_from_handle( *proc_ents.begin() ) )
3956 {
3957 Range::iterator lower = proc_ents.lower_bound( CN::TypeDimensionMap[0].first ),
3958 upper = proc_ents.upper_bound( CN::TypeDimensionMap[resolve_dim - 1].second );
3959 proc_ents.erase( lower, upper );
3960 }
3961
3962
3963
3964 return resolve_shared_ents( this_set, proc_ents, resolve_dim, shared_dim, NULL, id_tag );
3965 }
3966
3967 ErrorCode ParallelComm::resolve_shared_ents( EntityHandle this_set,
3968 Range& proc_ents,
3969 int resolve_dim,
3970 int shared_dim,
3971 Range* skin_ents,
3972 const Tag* id_tag )
3973 {
3974 #ifdef MOAB_HAVE_MPE
3975 if( myDebug->get_verbosity() == 2 )
3976 {
3977 define_mpe();
3978 MPE_Log_event( RESOLVE_START, procConfig.proc_rank(), "Entering resolve_shared_ents." );
3979 }
3980 #endif
3981
3982 ErrorCode result;
3983 myDebug->tprintf( 1, "Resolving shared entities.\n" );
3984
3985 if( resolve_dim < shared_dim )
3986 {
3987 MB_SET_ERR( MB_FAILURE, "MOAB does not support vertex-based partitions, only element-based ones" );
3988 }
3989
3990 if( -1 == shared_dim )
3991 {
3992 if( !proc_ents.empty() )
3993 shared_dim = mbImpl->dimension_from_handle( *proc_ents.begin() ) - 1;
3994 else if( resolve_dim == 3 )
3995 shared_dim = 2;
3996 }
3997 int max_global_resolve_dim = -1;
3998 int err = MPI_Allreduce( &resolve_dim, &max_global_resolve_dim, 1, MPI_INT, MPI_MAX, proc_config().proc_comm() );
3999 if( MPI_SUCCESS != err )
4000 {
4001 MB_SET_ERR( MB_FAILURE, "Unable to guess global resolve_dim" );
4002 }
4003 if( shared_dim < 0 || resolve_dim < 0 )
4004 {
4005
4006 resolve_dim = max_global_resolve_dim;
4007 shared_dim = resolve_dim - 1;
4008 }
4009
4010 if( resolve_dim < 0 || shared_dim < 0 ) return MB_SUCCESS;
4011
4012
4013
4014 Range tmp_skin_ents[4];
4015
4016
4017
4018 int skin_dim = resolve_dim - 1;
4019 if( !skin_ents )
4020 {
4021 skin_ents = tmp_skin_ents;
4022 skin_ents[resolve_dim] = proc_ents;
4023 Skinner skinner( mbImpl );
4024 result =
4025 skinner.find_skin( this_set, skin_ents[skin_dim + 1], false, skin_ents[skin_dim], NULL, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
4026 myDebug->tprintf( 1, "Found skin: skin_dim: %d resolve_dim: %d , now resolving.\n", skin_dim, resolve_dim );
4027 myDebug->tprintf( 3, "skin_ents[0].size(): %d skin_ents[1].size(): %d \n", (int)skin_ents[0].size(),
4028 (int)skin_ents[1].size() );
4029
4030 for( int this_dim = skin_dim - 1; this_dim >= 0; this_dim-- )
4031 {
4032 result =
4033 mbImpl->get_adjacencies( skin_ents[skin_dim], this_dim, true, skin_ents[this_dim], Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
4034
4035 if( this_set && skin_dim == 2 && this_dim == 1 )
4036 {
4037 result = mbImpl->add_entities( this_set, skin_ents[this_dim] );MB_CHK_ERR( result );
4038 }
4039 }
4040 }
4041 else if( skin_ents[resolve_dim].empty() )
4042 skin_ents[resolve_dim] = proc_ents;
4043
4044
4045 Tag gid_tag;
4046 if( id_tag )
4047 gid_tag = *id_tag;
4048 else
4049 {
4050 bool tag_created = false;
4051 int def_val = -1;
4052 result = mbImpl->tag_get_handle( GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE | MB_TAG_CREAT,
4053 &def_val, &tag_created );
4054 if( MB_ALREADY_ALLOCATED != result && MB_SUCCESS != result )
4055 {
4056 MB_SET_ERR( result, "Failed to create/get gid tag handle" );
4057 }
4058 else if( tag_created )
4059 {
4060
4061 result = assign_global_ids( this_set, skin_dim + 1, true, true, true );MB_CHK_SET_ERR( result, "Failed to assign global ids" );
4062 }
4063 }
4064
4065 DataType tag_type;
4066 result = mbImpl->tag_get_data_type( gid_tag, tag_type );MB_CHK_SET_ERR( result, "Failed to get tag data type" );
4067 int bytes_per_tag;
4068 result = mbImpl->tag_get_bytes( gid_tag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed to get number of bytes per tag" );
4069
4070
4071
4072
4073 std::vector< long > lgid_data( skin_ents[0].size() );
4074
4075
4076 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
4077 {
4078 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &lgid_data[0] );MB_CHK_SET_ERR( result, "Couldn't get gid tag for skin vertices" );
4079 }
4080 else if( 4 == bytes_per_tag )
4081 {
4082 std::vector< int > gid_data( lgid_data.size() );
4083 result = mbImpl->tag_get_data( gid_tag, skin_ents[0], &gid_data[0] );MB_CHK_SET_ERR( result, "Failed to get gid tag for skin vertices" );
4084 std::copy( gid_data.begin(), gid_data.end(), lgid_data.begin() );
4085 }
4086 else
4087 {
4088
4089 MB_SET_ERR( MB_FAILURE, "Unsupported id tag" );
4090 }
4091
4092
4093 std::vector< Ulong > handle_vec;
4094 std::copy( skin_ents[0].begin(), skin_ents[0].end(), std::back_inserter( handle_vec ) );
4095
4096 #ifdef MOAB_HAVE_MPE
4097 if( myDebug->get_verbosity() == 2 )
4098 {
4099 MPE_Log_event( SHAREDV_START, procConfig.proc_rank(), "Creating crystal router." );
4100 }
4101 #endif
4102
4103
4104 gs_data::crystal_data* cd = procConfig.crystal_router();
4105
4106
4118
4119 gs_data* gsd = new gs_data();
4120
4121 result = gsd->initialize( skin_ents[0].size(), &lgid_data[0], &handle_vec[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
4122
4123
4124 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
4125 result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags" );
4126
4127
4128 TupleList shared_verts;
4129 shared_verts.initialize( 2, 0, 1, 0, skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 ) );
4130 shared_verts.enableWriteAccess();
4131
4132 unsigned int i = 0, j = 0;
4133 for( unsigned int p = 0; p < gsd->nlinfo->_np; p++ )
4134 for( unsigned int np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
4135 {
4136 shared_verts.vi_wr[i++] = gsd->nlinfo->_sh_ind[j];
4137 shared_verts.vi_wr[i++] = gsd->nlinfo->_target[p];
4138 shared_verts.vul_wr[j] = gsd->nlinfo->_ulabels[j];
4139 j++;
4140 shared_verts.inc_n();
4141 }
4142
4143 myDebug->tprintf( 3, " shared verts size %d \n", (int)shared_verts.get_n() );
4144
4145 int max_size = skin_ents[0].size() * ( MAX_SHARING_PROCS + 1 );
4146 moab::TupleList::buffer sort_buffer;
4147 sort_buffer.buffer_init( max_size );
4148 shared_verts.sort( 0, &sort_buffer );
4149 sort_buffer.reset();
4150
4151
4152 int maxp = -1;
4153 std::vector< int > sharing_procs( MAX_SHARING_PROCS );
4154 std::fill( sharing_procs.begin(), sharing_procs.end(), maxp );
4155 j = 0;
4156 i = 0;
4157
4158
4159 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
4160 Range proc_verts;
4161 result = mbImpl->get_adjacencies( proc_ents, 0, false, proc_verts, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get proc_verts" );
4162
4163 myDebug->print( 3, " resolve shared ents: proc verts ", proc_verts );
4164 result = tag_shared_verts( shared_verts, skin_ents, proc_nvecs, proc_verts );MB_CHK_SET_ERR( result, "Failed to tag shared verts" );
4165
4166 #ifdef MOAB_HAVE_MPE
4167 if( myDebug->get_verbosity() == 2 )
4168 {
4169 MPE_Log_event( SHAREDV_END, procConfig.proc_rank(), "Finished tag_shared_verts." );
4170 }
4171 #endif
4172
4173
4174 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );MB_CHK_SET_ERR( result, "Failed to tag shared entities" );
4175
4176 shared_verts.reset();
4177
4178 if( myDebug->get_verbosity() > 0 )
4179 {
4180 for( std::map< std::vector< int >, std::vector< EntityHandle > >::const_iterator mit = proc_nvecs.begin();
4181 mit != proc_nvecs.end(); ++mit )
4182 {
4183 myDebug->tprintf( 1, "Iface: " );
4184 for( std::vector< int >::const_iterator vit = ( mit->first ).begin(); vit != ( mit->first ).end(); ++vit )
4185 myDebug->printf( 1, " %d", *vit );
4186 myDebug->print( 1, "\n" );
4187 }
4188 }
4189
4190
4191
4192 Range iface_sets;
4193 result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" );
4194
4195
4196 std::set< unsigned int > procs;
4197 result = get_interface_procs( procs, true );MB_CHK_SET_ERR( result, "Failed to get interface procs" );
4198
4199 #ifndef NDEBUG
4200 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Shared handle check failed after interface vertex exchange" );
4201 #endif
4202
4203
4204
4205 result = exchange_ghost_cells( -1, -1, 0, 0, true, true );MB_CHK_SET_ERR( result, "Failed to resolve shared entity remote handles" );
4206
4207
4208 result = create_iface_pc_links();MB_CHK_SET_ERR( result, "Failed to create interface parent/child links" );
4209
4210 gsd->reset();
4211 delete gsd;
4212
4213 #ifdef MOAB_HAVE_MPE
4214 if( myDebug->get_verbosity() == 2 )
4215 {
4216 MPE_Log_event( RESOLVE_END, procConfig.proc_rank(), "Exiting resolve_shared_ents." );
4217 }
4218 #endif
4219
4220
4221
4222
4223
4224
4225 return result;
4226 }
4227
4228 void ParallelComm::define_mpe()
4229 {
4230 #ifdef MOAB_HAVE_MPE
4231 if( myDebug->get_verbosity() == 2 )
4232 {
4233
4234 int success;
4235 MPE_Log_get_state_eventIDs( &IFACE_START, &IFACE_END );
4236 MPE_Log_get_state_eventIDs( &GHOST_START, &GHOST_END );
4237 MPE_Log_get_state_eventIDs( &SHAREDV_START, &SHAREDV_END );
4238 MPE_Log_get_state_eventIDs( &RESOLVE_START, &RESOLVE_END );
4239 MPE_Log_get_state_eventIDs( &ENTITIES_START, &ENTITIES_END );
4240 MPE_Log_get_state_eventIDs( &RHANDLES_START, &RHANDLES_END );
4241 MPE_Log_get_state_eventIDs( &OWNED_START, &OWNED_END );
4242 success = MPE_Describe_state( IFACE_START, IFACE_END, "Resolve interface ents", "green" );
4243 assert( MPE_LOG_OK == success );
4244 success = MPE_Describe_state( GHOST_START, GHOST_END, "Exchange ghost ents", "red" );
4245 assert( MPE_LOG_OK == success );
4246 success = MPE_Describe_state( SHAREDV_START, SHAREDV_END, "Resolve interface vertices", "blue" );
4247 assert( MPE_LOG_OK == success );
4248 success = MPE_Describe_state( RESOLVE_START, RESOLVE_END, "Resolve shared ents", "purple" );
4249 assert( MPE_LOG_OK == success );
4250 success = MPE_Describe_state( ENTITIES_START, ENTITIES_END, "Exchange shared ents", "yellow" );
4251 assert( MPE_LOG_OK == success );
4252 success = MPE_Describe_state( RHANDLES_START, RHANDLES_END, "Remote handles", "cyan" );
4253 assert( MPE_LOG_OK == success );
4254 success = MPE_Describe_state( OWNED_START, OWNED_END, "Exchange owned ents", "black" );
4255 assert( MPE_LOG_OK == success );
4256 }
4257 #endif
4258 }
4259
4260 ErrorCode ParallelComm::resolve_shared_ents( ParallelComm** pc,
4261 const unsigned int np,
4262 EntityHandle this_set,
4263 const int part_dim )
4264 {
4265 std::vector< Range > verts( np );
4266 int tot_verts = 0;
4267 unsigned int p, i, j, v;
4268 ErrorCode rval;
4269 for( p = 0; p < np; p++ )
4270 {
4271 Skinner skinner( pc[p]->get_moab() );
4272 Range part_ents, skin_ents;
4273 rval = pc[p]->get_moab()->get_entities_by_dimension( this_set, part_dim, part_ents );
4274 if( MB_SUCCESS != rval ) return rval;
4275 rval = skinner.find_skin( this_set, part_ents, false, skin_ents, 0, true, true, true );
4276 if( MB_SUCCESS != rval ) return rval;
4277 rval = pc[p]->get_moab()->get_adjacencies( skin_ents, 0, true, verts[p], Interface::UNION );
4278 if( MB_SUCCESS != rval ) return rval;
4279 tot_verts += verts[p].size();
4280 }
4281
4282 TupleList shared_ents;
4283 shared_ents.initialize( 2, 0, 1, 0, tot_verts );
4284 shared_ents.enableWriteAccess();
4285
4286 i = 0;
4287 j = 0;
4288 std::vector< int > gids;
4289 Range::iterator rit;
4290 Tag gid_tag;
4291 for( p = 0; p < np; p++ )
4292 {
4293 gid_tag = pc[p]->get_moab()->globalId_tag();
4294
4295 gids.resize( verts[p].size() );
4296 rval = pc[p]->get_moab()->tag_get_data( gid_tag, verts[p], &gids[0] );
4297 if( MB_SUCCESS != rval ) return rval;
4298
4299 for( v = 0, rit = verts[p].begin(); v < gids.size(); v++, ++rit )
4300 {
4301 shared_ents.vi_wr[i++] = gids[v];
4302 shared_ents.vi_wr[i++] = p;
4303 shared_ents.vul_wr[j] = *rit;
4304 j++;
4305 shared_ents.inc_n();
4306 }
4307 }
4308
4309 moab::TupleList::buffer sort_buffer;
4310 sort_buffer.buffer_init( tot_verts );
4311 shared_ents.sort( 0, &sort_buffer );
4312 sort_buffer.reset();
4313
4314 j = 0;
4315 i = 0;
4316 std::vector< EntityHandle > handles;
4317 std::vector< int > procs;
4318
4319 while( i < shared_ents.get_n() )
4320 {
4321 handles.clear();
4322 procs.clear();
4323
4324
4325 int this_gid = shared_ents.vi_rd[j];
4326 while( i < shared_ents.get_n() && shared_ents.vi_rd[j] == this_gid )
4327 {
4328 j++;
4329 procs.push_back( shared_ents.vi_rd[j++] );
4330 handles.push_back( shared_ents.vul_rd[i++] );
4331 }
4332 if( 1 == procs.size() ) continue;
4333
4334 for( v = 0; v < procs.size(); v++ )
4335 {
4336 rval = pc[procs[v]]->update_remote_data( handles[v], &procs[0], &handles[0], procs.size(),
4337 ( procs[0] == (int)pc[procs[v]]->rank()
4338 ? PSTATUS_INTERFACE
4339 : ( PSTATUS_NOT_OWNED | PSTATUS_INTERFACE ) ) );
4340 if( MB_SUCCESS != rval ) return rval;
4341 }
4342 }
4343
4344 std::set< unsigned int > psets;
4345 for( p = 0; p < np; p++ )
4346 {
4347 rval = pc[p]->create_interface_sets( this_set, part_dim, part_dim - 1 );
4348 if( MB_SUCCESS != rval ) return rval;
4349
4350 psets.clear();
4351 rval = pc[p]->get_interface_procs( psets, true );
4352 if( MB_SUCCESS != rval ) return rval;
4353 }
4354
4355 shared_ents.reset();
4356
4357 return MB_SUCCESS;
4358 }
4359
4360 ErrorCode ParallelComm::tag_iface_entities()
4361 {
4362 ErrorCode result = MB_SUCCESS;
4363 Range iface_ents, tmp_ents, rmv_ents;
4364 std::vector< unsigned char > pstat;
4365 unsigned char set_pstat;
4366 Range::iterator rit2;
4367 unsigned int i;
4368
4369 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
4370 {
4371 iface_ents.clear();
4372
4373 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get interface set contents" );
4374 pstat.resize( iface_ents.size() );
4375 result = mbImpl->tag_get_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set entities" );
4376 result = mbImpl->tag_get_data( pstatus_tag(), &( *rit ), 1, &set_pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus values for interface set" );
4377 rmv_ents.clear();
4378 for( rit2 = iface_ents.begin(), i = 0; rit2 != iface_ents.end(); ++rit2, i++ )
4379 {
4380 if( !( pstat[i] & PSTATUS_INTERFACE ) )
4381 {
4382 rmv_ents.insert( *rit2 );
4383 pstat[i] = 0x0;
4384 }
4385 }
4386 result = mbImpl->remove_entities( *rit, rmv_ents );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
4387
4388 if( !( set_pstat & PSTATUS_NOT_OWNED ) ) continue;
4389
4390
4391
4392 iface_ents = subtract( iface_ents, rmv_ents );
4393
4394 std::remove_if( pstat.begin(), pstat.end(),
4395 std::bind( std::equal_to< unsigned char >(), std::placeholders::_1, 0x0 ) );
4396
4397
4398
4399 unsigned int sz = iface_ents.size();
4400 for( i = 0; i < sz; i++ )
4401 pstat[i] |= PSTATUS_NOT_OWNED;
4402
4403
4404 result = mbImpl->tag_set_data( pstatus_tag(), iface_ents, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus values for interface set entities" );
4405 }
4406
4407 return MB_SUCCESS;
4408 }
4409
4410 ErrorCode ParallelComm::set_pstatus_entities( Range& pstatus_ents,
4411 unsigned char pstatus_val,
4412 bool lower_dim_ents,
4413 bool verts_too,
4414 int operation )
4415 {
4416 std::vector< unsigned char > pstatus_vals( pstatus_ents.size() );
4417 Range all_ents, *range_ptr = &pstatus_ents;
4418 ErrorCode result;
4419 if( lower_dim_ents || verts_too )
4420 {
4421 all_ents = pstatus_ents;
4422 range_ptr = &all_ents;
4423 int start_dim = ( lower_dim_ents ? mbImpl->dimension_from_handle( *pstatus_ents.rbegin() ) - 1 : 0 );
4424 for( ; start_dim >= 0; start_dim-- )
4425 {
4426 result = mbImpl->get_adjacencies( all_ents, start_dim, true, all_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get adjacencies for pstatus entities" );
4427 }
4428 }
4429 if( Interface::UNION == operation )
4430 {
4431 result = mbImpl->tag_get_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
4432 for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
4433 pstatus_vals[i] |= pstatus_val;
4434 }
4435 else
4436 {
4437 for( unsigned int i = 0; i < pstatus_vals.size(); i++ )
4438 pstatus_vals[i] = pstatus_val;
4439 }
4440 result = mbImpl->tag_set_data( pstatus_tag(), *range_ptr, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
4441
4442 return MB_SUCCESS;
4443 }
4444
4445 ErrorCode ParallelComm::set_pstatus_entities( EntityHandle* pstatus_ents,
4446 int num_ents,
4447 unsigned char pstatus_val,
4448 bool lower_dim_ents,
4449 bool verts_too,
4450 int operation )
4451 {
4452 std::vector< unsigned char > pstatus_vals( num_ents );
4453 ErrorCode result;
4454 if( lower_dim_ents || verts_too )
4455 {
4456
4457 Range tmp_range;
4458 std::copy( pstatus_ents, pstatus_ents + num_ents, range_inserter( tmp_range ) );
4459 return set_pstatus_entities( tmp_range, pstatus_val, lower_dim_ents, verts_too, operation );
4460 }
4461
4462 if( Interface::UNION == operation )
4463 {
4464 result = mbImpl->tag_get_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
4465 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
4466 pstatus_vals[i] |= pstatus_val;
4467 }
4468 else
4469 {
4470 for( unsigned int i = 0; i < (unsigned int)num_ents; i++ )
4471 pstatus_vals[i] = pstatus_val;
4472 }
4473 result = mbImpl->tag_set_data( pstatus_tag(), pstatus_ents, num_ents, &pstatus_vals[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
4474
4475 return MB_SUCCESS;
4476 }
4477
4478 static size_t choose_owner_idx( const std::vector< unsigned >& proc_list )
4479 {
4480
4481
4482 unsigned val = 0;
4483 for( size_t i = 0; i < proc_list.size(); i++ )
4484 val ^= proc_list[i];
4485 srand( (int)( val ) );
4486 return rand() % proc_list.size();
4487 }
4488
4489 struct set_tuple
4490 {
4491 unsigned idx;
4492 unsigned proc;
4493 EntityHandle handle;
4494 inline bool operator<( set_tuple other ) const
4495 {
4496 return ( idx == other.idx ) ? ( proc < other.proc ) : ( idx < other.idx );
4497 }
4498 };
4499
4500 ErrorCode ParallelComm::resolve_shared_sets( EntityHandle file, const Tag* idtag )
4501 {
4502
4503 const char* const shared_set_tag_names[] = { GEOM_DIMENSION_TAG_NAME, MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME,
4504 NEUMANN_SET_TAG_NAME, PARALLEL_PARTITION_TAG_NAME };
4505 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
4506 Range candidate_sets;
4507 ErrorCode result = MB_FAILURE;
4508
4509
4510
4511 if( !idtag )
4512 {
4513 Tag gid, tag;
4514 gid = mbImpl->globalId_tag();
4515 if( NULL != gid ) result = mbImpl->tag_get_handle( GEOM_DIMENSION_TAG_NAME, 1, MB_TYPE_INTEGER, tag );
4516 if( MB_SUCCESS == result )
4517 {
4518 for( int d = 0; d < 4; d++ )
4519 {
4520 candidate_sets.clear();
4521 const void* vals[] = { &d };
4522 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, vals, 1, candidate_sets );
4523 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, gid );
4524 }
4525 }
4526
4527 for( int i = 1; i < num_tags; i++ )
4528 {
4529 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag );
4530 if( MB_SUCCESS == result )
4531 {
4532 candidate_sets.clear();
4533 result = mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets );
4534 if( MB_SUCCESS == result ) resolve_shared_sets( candidate_sets, tag );
4535 }
4536 }
4537
4538 return MB_SUCCESS;
4539 }
4540
4541 for( int i = 0; i < num_tags; i++ )
4542 {
4543 Tag tag;
4544 result = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tag, MB_TAG_ANY );
4545 if( MB_SUCCESS != result ) continue;
4546
4547 mbImpl->get_entities_by_type_and_tag( file, MBENTITYSET, &tag, 0, 1, candidate_sets, Interface::UNION );
4548 }
4549
4550
4551 Range::iterator hint = candidate_sets.begin();
4552 Range all_sets;
4553 mbImpl->get_entities_by_type( file, MBENTITYSET, all_sets );
4554 all_sets = subtract( all_sets, candidate_sets );
4555 Range::iterator it = all_sets.begin();
4556 while( it != all_sets.end() )
4557 {
4558 Range contents;
4559 mbImpl->get_entities_by_handle( *it, contents );
4560 contents.erase( contents.lower_bound( MBENTITYSET ), contents.end() );
4561 filter_pstatus( contents, PSTATUS_SHARED, PSTATUS_OR );
4562 if( contents.empty() )
4563 {
4564 ++it;
4565 }
4566 else
4567 {
4568 hint = candidate_sets.insert( hint, *it );
4569 it = all_sets.erase( it );
4570 }
4571 }
4572
4573
4574 Range prev_list = candidate_sets;
4575 while( !prev_list.empty() )
4576 {
4577 it = all_sets.begin();
4578 Range new_list;
4579 hint = new_list.begin();
4580 while( it != all_sets.end() )
4581 {
4582 Range contents;
4583 mbImpl->get_entities_by_type( *it, MBENTITYSET, contents );
4584 if( !intersect( prev_list, contents ).empty() )
4585 {
4586 hint = new_list.insert( hint, *it );
4587 it = all_sets.erase( it );
4588 }
4589 else
4590 {
4591 new_list.clear();
4592 mbImpl->get_child_meshsets( *it, contents );
4593 if( !intersect( prev_list, contents ).empty() )
4594 {
4595 hint = new_list.insert( hint, *it );
4596 it = all_sets.erase( it );
4597 }
4598 else
4599 {
4600 ++it;
4601 }
4602 }
4603 }
4604
4605 candidate_sets.merge( new_list );
4606 prev_list.swap( new_list );
4607 }
4608
4609 return resolve_shared_sets( candidate_sets, *idtag );
4610 }
4611
4612 #ifndef NDEBUG
4613 bool is_sorted_unique( std::vector< unsigned >& v )
4614 {
4615 for( size_t i = 1; i < v.size(); i++ )
4616 if( v[i - 1] >= v[i] ) return false;
4617 return true;
4618 }
4619 #endif
4620
4621 ErrorCode ParallelComm::resolve_shared_sets( Range& sets, Tag idtag )
4622 {
4623 ErrorCode result;
4624 const unsigned rk = proc_config().proc_rank();
4625 MPI_Comm cm = proc_config().proc_comm();
4626
4627
4628
4629
4630 std::vector< long > larray;
4631 std::vector< Ulong > handles;
4632 Range tmp_sets;
4633
4634
4635
4636
4637
4638 DataType tag_type;
4639 result = mbImpl->tag_get_data_type( idtag, tag_type );MB_CHK_SET_ERR( result, "Failed getting tag data type" );
4640 int bytes_per_tag;
4641 result = mbImpl->tag_get_bytes( idtag, bytes_per_tag );MB_CHK_SET_ERR( result, "Failed getting number of bytes per tag" );
4642
4643
4644
4645 for( Range::iterator rit = sets.begin(); rit != sets.end(); ++rit )
4646 {
4647 if( sizeof( long ) == bytes_per_tag && ( ( MB_TYPE_HANDLE == tag_type ) || ( MB_TYPE_OPAQUE == tag_type ) ) )
4648 {
4649 long dum;
4650 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
4651 if( MB_SUCCESS == result )
4652 {
4653 larray.push_back( dum );
4654 handles.push_back( *rit );
4655 tmp_sets.insert( tmp_sets.end(), *rit );
4656 }
4657 }
4658 else if( 4 == bytes_per_tag )
4659 {
4660 int dum;
4661 result = mbImpl->tag_get_data( idtag, &( *rit ), 1, &dum );
4662 if( MB_SUCCESS == result )
4663 {
4664 larray.push_back( dum );
4665 handles.push_back( *rit );
4666 tmp_sets.insert( tmp_sets.end(), *rit );
4667 }
4668 }
4669 }
4670
4671 const size_t nsets = handles.size();
4672
4673
4674
4675
4676
4677
4678 gs_data::crystal_data* cd = procConfig.crystal_router();
4679 gs_data* gsd = new gs_data();
4680 result = gsd->initialize( nsets, &larray[0], &handles[0], 2, 1, 1, cd );MB_CHK_SET_ERR( result, "Failed to create gs data" );
4681
4682
4683
4684
4685
4686 size_t ntuple = 0;
4687 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
4688 ntuple += gsd->nlinfo->_nshared[p];
4689 std::vector< set_tuple > tuples;
4690 tuples.reserve( ntuple );
4691 size_t j = 0;
4692 for( unsigned p = 0; p < gsd->nlinfo->_np; p++ )
4693 {
4694 for( unsigned np = 0; np < gsd->nlinfo->_nshared[p]; np++ )
4695 {
4696 set_tuple t;
4697 t.idx = gsd->nlinfo->_sh_ind[j];
4698 t.proc = gsd->nlinfo->_target[p];
4699 t.handle = gsd->nlinfo->_ulabels[j];
4700 tuples.push_back( t );
4701 j++;
4702 }
4703 }
4704 std::sort( tuples.begin(), tuples.end() );
4705
4706
4707 gsd->reset();
4708 delete gsd;
4709
4710
4711 size_t ti = 0;
4712 unsigned idx = 0;
4713 std::vector< unsigned > procs;
4714 Range::iterator si = tmp_sets.begin();
4715 while( si != tmp_sets.end() && ti < tuples.size() )
4716 {
4717 assert( idx <= tuples[ti].idx );
4718 if( idx < tuples[ti].idx ) si += ( tuples[ti].idx - idx );
4719 idx = tuples[ti].idx;
4720
4721 procs.clear();
4722 size_t ti_init = ti;
4723 while( ti < tuples.size() && tuples[ti].idx == idx )
4724 {
4725 procs.push_back( tuples[ti].proc );
4726 ++ti;
4727 }
4728 assert( is_sorted_unique( procs ) );
4729
4730 result = sharedSetData->set_sharing_procs( *si, procs );
4731 if( MB_SUCCESS != result )
4732 {
4733 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
4734 std::cerr.flush();
4735 MPI_Abort( cm, 1 );
4736 }
4737
4738
4739
4740 std::vector< unsigned >::iterator it = std::lower_bound( procs.begin(), procs.end(), rk );
4741 assert( it == procs.end() || *it > rk );
4742 procs.insert( it, rk );
4743 size_t owner_idx = choose_owner_idx( procs );
4744 EntityHandle owner_handle;
4745 if( procs[owner_idx] == rk )
4746 owner_handle = *si;
4747 else if( procs[owner_idx] > rk )
4748 owner_handle = tuples[ti_init + owner_idx - 1].handle;
4749 else
4750 owner_handle = tuples[ti_init + owner_idx].handle;
4751 result = sharedSetData->set_owner( *si, procs[owner_idx], owner_handle );
4752 if( MB_SUCCESS != result )
4753 {
4754 std::cerr << "Failure at " __FILE__ ":" << __LINE__ << std::endl;
4755 std::cerr.flush();
4756 MPI_Abort( cm, 1 );
4757 }
4758
4759 ++si;
4760 ++idx;
4761 }
4762
4763 return MB_SUCCESS;
4764 }
4765
4766 ErrorCode ParallelComm::augment_default_sets_with_ghosts( EntityHandle file_set )
4767 {
4768
4769
4770
4771
4772 if( procConfig.proc_size() < 2 ) return MB_SUCCESS;
4773 const char* const shared_set_tag_names[] = { MATERIAL_SET_TAG_NAME, DIRICHLET_SET_TAG_NAME, NEUMANN_SET_TAG_NAME,
4774 PARALLEL_PARTITION_TAG_NAME };
4775
4776 int num_tags = sizeof( shared_set_tag_names ) / sizeof( shared_set_tag_names[0] );
4777
4778 Range* rangeSets = new Range[num_tags];
4779 Tag* tags = new Tag[num_tags + 1];
4780
4781 int my_rank = rank();
4782 int** tagVals = new int*[num_tags];
4783 for( int i = 0; i < num_tags; i++ )
4784 tagVals[i] = NULL;
4785 ErrorCode rval;
4786
4787
4788
4789
4790
4791 typedef std::map< int, EntityHandle > MVal;
4792 typedef std::map< int, EntityHandle >::iterator itMVal;
4793 MVal* localMaps = new MVal[num_tags];
4794
4795 for( int i = 0; i < num_tags; i++ )
4796 {
4797
4798 rval = mbImpl->tag_get_handle( shared_set_tag_names[i], 1, MB_TYPE_INTEGER, tags[i], MB_TAG_ANY );
4799 if( MB_SUCCESS != rval ) continue;
4800 rval = mbImpl->get_entities_by_type_and_tag( file_set, MBENTITYSET, &( tags[i] ), 0, 1, rangeSets[i],
4801 Interface::UNION );MB_CHK_SET_ERR( rval, "can't get sets with a tag" );
4802
4803 if( rangeSets[i].size() > 0 )
4804 {
4805 tagVals[i] = new int[rangeSets[i].size()];
4806
4807 rval = mbImpl->tag_get_data( tags[i], rangeSets[i], tagVals[i] );MB_CHK_SET_ERR( rval, "can't get set tag values" );
4808
4809 for( int j = 0; j < (int)rangeSets[i].size(); j++ )
4810 {
4811 localMaps[i][tagVals[i][j]] = rangeSets[i][j];
4812 }
4813 }
4814 }
4815
4816 tags[num_tags] = mbImpl->globalId_tag();
4817
4818 TupleList remoteEnts;
4819
4820
4821
4822
4823
4824 int initialSize = (int)sharedEnts.size();
4825
4826
4827
4828
4829 remoteEnts.initialize( 3, 0, 1, 0, initialSize );
4830 remoteEnts.enableWriteAccess();
4831
4832
4833
4834
4835 std::set< EntityHandle > own_and_sha;
4836 int ir = 0, jr = 0;
4837 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
4838 {
4839
4840 EntityHandle geh = *vit;
4841 if( own_and_sha.find( geh ) != own_and_sha.end() )
4842 continue;
4843 int procs[MAX_SHARING_PROCS];
4844 EntityHandle handles[MAX_SHARING_PROCS];
4845 int nprocs;
4846 unsigned char pstat;
4847 rval = get_sharing_data( geh, procs, handles, pstat, nprocs );
4848 if( rval != MB_SUCCESS )
4849 {
4850 for( int i = 0; i < num_tags; i++ )
4851 delete[] tagVals[i];
4852 delete[] tagVals;
4853
4854 MB_CHK_SET_ERR( rval, "Failed to get sharing data" );
4855 }
4856 if( pstat & PSTATUS_NOT_OWNED ) continue;
4857 own_and_sha.insert( geh );
4858 for( int i = 0; i < num_tags; i++ )
4859 {
4860 for( int j = 0; j < (int)rangeSets[i].size(); j++ )
4861 {
4862 EntityHandle specialSet = rangeSets[i][j];
4863 if( mbImpl->contains_entities( specialSet, &geh, 1 ) )
4864 {
4865
4866
4867 for( int k = 0; k < nprocs; k++ )
4868 {
4869 if( procs[k] != my_rank )
4870 {
4871 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
4872 {
4873
4874 int oldSize = remoteEnts.get_max();
4875
4876 remoteEnts.resize( oldSize + oldSize / 2 + 1 );
4877 }
4878 remoteEnts.vi_wr[ir++] = procs[k];
4879 remoteEnts.vi_wr[ir++] = i;
4880 remoteEnts.vi_wr[ir++] = tagVals[i][j];
4881 remoteEnts.vul_wr[jr++] = handles[k];
4882 remoteEnts.inc_n();
4883 }
4884 }
4885 }
4886 }
4887 }
4888
4889
4890 int gid;
4891 rval = mbImpl->tag_get_data( tags[num_tags], &geh, 1, &gid );MB_CHK_SET_ERR( rval, "Failed to get global id" );
4892 if( gid != 0 )
4893 {
4894 for( int k = 0; k < nprocs; k++ )
4895 {
4896 if( procs[k] != my_rank )
4897 {
4898 if( remoteEnts.get_n() >= remoteEnts.get_max() - 1 )
4899 {
4900
4901 int oldSize = remoteEnts.get_max();
4902
4903 remoteEnts.resize( oldSize + oldSize / 2 + 1 );
4904 }
4905 remoteEnts.vi_wr[ir++] = procs[k];
4906 remoteEnts.vi_wr[ir++] = num_tags;
4907 remoteEnts.vi_wr[ir++] = gid;
4908 remoteEnts.vul_wr[jr++] = handles[k];
4909 remoteEnts.inc_n();
4910 }
4911 }
4912 }
4913 }
4914
4915 #ifndef NDEBUG
4916 if( my_rank == 1 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 1, before augment routing" );
4917 MPI_Barrier( procConfig.proc_comm() );
4918 int sentEnts = remoteEnts.get_n();
4919 assert( ( sentEnts == jr ) && ( 3 * sentEnts == ir ) );
4920 #endif
4921
4922 gs_data::crystal_data* cd = this->procConfig.crystal_router();
4923
4924
4925 rval = cd->gs_transfer( 1, remoteEnts, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" );
4926 #ifndef NDEBUG
4927 if( my_rank == 0 && 1 == get_debug_verbosity() ) remoteEnts.print( " on rank 0, after augment routing" );
4928 MPI_Barrier( procConfig.proc_comm() );
4929 #endif
4930
4931
4932 int received = remoteEnts.get_n();
4933 for( int i = 0; i < received; i++ )
4934 {
4935
4936 EntityHandle geh = (EntityHandle)remoteEnts.vul_rd[i];
4937 int from_proc = remoteEnts.vi_rd[3 * i];
4938 if( my_rank == from_proc )
4939 std::cout << " unexpected receive from my rank " << my_rank << " during augmenting with ghosts\n ";
4940 int tag_type = remoteEnts.vi_rd[3 * i + 1];
4941 assert( ( 0 <= tag_type ) && ( tag_type <= num_tags ) );
4942 int value = remoteEnts.vi_rd[3 * i + 2];
4943 if( tag_type == num_tags )
4944 {
4945
4946 rval = mbImpl->tag_set_data( tags[num_tags], &geh, 1, &value );MB_CHK_SET_ERR( rval, "Error in setting gid tag" );
4947 }
4948 else
4949 {
4950
4951 MVal& lmap = localMaps[tag_type];
4952 itMVal itm = lmap.find( value );
4953 if( itm == lmap.end() )
4954 {
4955
4956 EntityHandle newSet;
4957 rval = mbImpl->create_meshset( MESHSET_SET, newSet );MB_CHK_SET_ERR( rval, "can't create new set" );
4958 lmap[value] = newSet;
4959
4960 rval = mbImpl->tag_set_data( tags[tag_type], &newSet, 1, &value );MB_CHK_SET_ERR( rval, "can't set tag for new set" );
4961
4962
4963 if( file_set )
4964 {
4965 rval = mbImpl->add_entities( file_set, &newSet, 1 );MB_CHK_SET_ERR( rval, "can't add new set to the file set" );
4966 }
4967 }
4968
4969 rval = mbImpl->add_entities( lmap[value], &geh, 1 );MB_CHK_SET_ERR( rval, "can't add ghost ent to the set" );
4970 }
4971 }
4972
4973 for( int i = 0; i < num_tags; i++ )
4974 delete[] tagVals[i];
4975 delete[] tagVals;
4976 delete[] rangeSets;
4977 delete[] tags;
4978 delete[] localMaps;
4979 return MB_SUCCESS;
4980 }
4981 ErrorCode ParallelComm::create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim )
4982 {
4983 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
4984
4985
4986 int procs[MAX_SHARING_PROCS];
4987 EntityHandle handles[MAX_SHARING_PROCS];
4988 ErrorCode result;
4989 int nprocs;
4990 unsigned char pstat;
4991 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
4992 {
4993 if( shared_dim != -1 && mbImpl->dimension_from_handle( *vit ) > shared_dim ) continue;
4994 result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
4995 std::sort( procs, procs + nprocs );
4996 std::vector< int > tmp_procs( procs, procs + nprocs );
4997 assert( tmp_procs.size() != 2 );
4998 proc_nvecs[tmp_procs].push_back( *vit );
4999 }
5000
5001 Skinner skinner( mbImpl );
5002 Range skin_ents[4];
5003 result = mbImpl->get_entities_by_dimension( this_set, resolve_dim, skin_ents[resolve_dim] );MB_CHK_SET_ERR( result, "Failed to get skin entities by dimension" );
5004 result =
5005 skinner.find_skin( this_set, skin_ents[resolve_dim], false, skin_ents[resolve_dim - 1], 0, true, true, true );MB_CHK_SET_ERR( result, "Failed to find skin" );
5006 if( shared_dim > 1 )
5007 {
5008 result = mbImpl->get_adjacencies( skin_ents[resolve_dim - 1], resolve_dim - 2, true, skin_ents[resolve_dim - 2],
5009 Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get skin adjacencies" );
5010 }
5011
5012 result = get_proc_nvecs( resolve_dim, shared_dim, skin_ents, proc_nvecs );
5013
5014 return create_interface_sets( proc_nvecs );
5015 }
5016
5017 ErrorCode ParallelComm::create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
5018 {
5019 if( proc_nvecs.empty() ) return MB_SUCCESS;
5020
5021 int proc_ids[MAX_SHARING_PROCS];
5022 EntityHandle proc_handles[MAX_SHARING_PROCS];
5023 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
5024 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in create_interface_sets" );
5025 Range::iterator rit;
5026
5027
5028 std::vector< unsigned char > pstatus;
5029 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator vit = proc_nvecs.begin();
5030 vit != proc_nvecs.end(); ++vit )
5031 {
5032
5033 EntityHandle new_set;
5034 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
5035 interfaceSets.insert( new_set );
5036
5037
5038 assert( !vit->second.empty() );
5039 result = mbImpl->add_entities( new_set, &( vit->second )[0], ( vit->second ).size() );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
5040
5041 if( vit->first.size() == 1 )
5042 {
5043 assert( ( vit->first )[0] != (int)procConfig.proc_rank() );
5044 result = mbImpl->tag_set_data( shp_tag, &new_set, 1, &( vit->first )[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
5045 proc_handles[0] = 0;
5046 result = mbImpl->tag_set_data( shh_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
5047 }
5048 else
5049 {
5050
5051 if( vit->first.size() > MAX_SHARING_PROCS )
5052 {
5053 std::cerr << "Exceeded MAX_SHARING_PROCS for " << CN::EntityTypeName( TYPE_FROM_HANDLE( new_set ) )
5054 << ' ' << ID_FROM_HANDLE( new_set ) << " on process " << proc_config().proc_rank()
5055 << std::endl;
5056 std::cerr.flush();
5057 MPI_Abort( proc_config().proc_comm(), 66 );
5058 }
5059
5060 std::copy( vit->first.begin(), vit->first.end(), proc_ids );
5061 std::fill( proc_ids + vit->first.size(), proc_ids + MAX_SHARING_PROCS, -1 );
5062 result = mbImpl->tag_set_data( shps_tag, &new_set, 1, proc_ids );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
5063 unsigned int ind = std::find( proc_ids, proc_ids + vit->first.size(), procConfig.proc_rank() ) - proc_ids;
5064 assert( ind < vit->first.size() );
5065 std::fill( proc_handles, proc_handles + MAX_SHARING_PROCS, 0 );
5066 proc_handles[ind] = new_set;
5067 result = mbImpl->tag_set_data( shhs_tag, &new_set, 1, proc_handles );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
5068 }
5069
5070
5071 int min_proc = ( vit->first )[0];
5072 unsigned char pval = ( PSTATUS_SHARED | PSTATUS_INTERFACE );
5073 if( min_proc < (int)procConfig.proc_rank() ) pval |= PSTATUS_NOT_OWNED;
5074 if( vit->first.size() > 1 ) pval |= PSTATUS_MULTISHARED;
5075 result = mbImpl->tag_set_data( pstat_tag, &new_set, 1, &pval );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
5076
5077
5078 pstatus.clear();
5079 std::vector< EntityHandle > verts;
5080 for( std::vector< EntityHandle >::iterator v2it = ( vit->second ).begin(); v2it != ( vit->second ).end();
5081 ++v2it )
5082 if( mbImpl->type_from_handle( *v2it ) == MBVERTEX ) verts.push_back( *v2it );
5083 pstatus.resize( verts.size(), pval );
5084 if( !verts.empty() )
5085 {
5086 result = mbImpl->tag_set_data( pstat_tag, &verts[0], verts.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to tag interface set vertices with pstatus" );
5087 }
5088 }
5089
5090 return MB_SUCCESS;
5091 }
5092
5093 ErrorCode ParallelComm::create_iface_pc_links()
5094 {
5095
5096
5097
5098
5099 Tag tmp_iface_tag;
5100 EntityHandle tmp_iface_set = 0;
5101 ErrorCode result = mbImpl->tag_get_handle( "__tmp_iface", 1, MB_TYPE_HANDLE, tmp_iface_tag,
5102 MB_TAG_DENSE | MB_TAG_CREAT, &tmp_iface_set );MB_CHK_SET_ERR( result, "Failed to create temporary interface set tag" );
5103
5104 Range iface_ents;
5105 std::vector< EntityHandle > tag_vals;
5106 Range::iterator rit;
5107
5108 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
5109 {
5110
5111 iface_ents.clear();
5112 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in interface set" );
5113
5114 if( iface_ents.empty() ) continue;
5115
5116 tag_vals.resize( iface_ents.size() );
5117 std::fill( tag_vals.begin(), tag_vals.end(), *rit );
5118 result = mbImpl->tag_set_data( tmp_iface_tag, iface_ents, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to tag iface entities with interface set" );
5119 }
5120
5121
5122 Range tmp_ents2;
5123 for( int d = 2; d >= 0; d-- )
5124 {
5125 for( rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
5126 {
5127
5128 iface_ents.clear();
5129 result = mbImpl->get_entities_by_handle( *rit, iface_ents, true );MB_CHK_SET_ERR( result, "Failed to get entities by handle" );
5130 if( iface_ents.empty() || mbImpl->dimension_from_handle( *iface_ents.rbegin() ) != d ) continue;
5131
5132
5133 result = mbImpl->get_adjacencies( &( *iface_ents.begin() ), 1, d + 1, false, tmp_ents2 );MB_CHK_SET_ERR( result, "Failed to get adjacencies for interface sets" );
5134 tag_vals.resize( tmp_ents2.size() );
5135 result = mbImpl->tag_get_data( tmp_iface_tag, tmp_ents2, &tag_vals[0] );MB_CHK_SET_ERR( result, "Failed to get tmp iface tag for interface sets" );
5136
5137
5138 EntityHandle last_set = 0;
5139 for( unsigned int i = 0; i < tag_vals.size(); i++ )
5140 {
5141 if( tag_vals[i] && tag_vals[i] != last_set )
5142 {
5143 result = mbImpl->add_parent_child( tag_vals[i], *rit );MB_CHK_SET_ERR( result, "Failed to add parent/child link for interface set" );
5144 last_set = tag_vals[i];
5145 }
5146 }
5147 }
5148 }
5149
5150
5151 result = mbImpl->tag_delete( tmp_iface_tag );MB_CHK_SET_ERR( result, "Failed to delete tmp iface tag" );
5152
5153 return MB_SUCCESS;
5154 }
5155
5156 ErrorCode ParallelComm::get_proc_nvecs( int resolve_dim,
5157 int shared_dim,
5158 Range* skin_ents,
5159 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs )
5160 {
5161
5162 ErrorCode result;
5163 const EntityHandle* connect;
5164 int num_connect;
5165 std::set< int > sharing_procs;
5166 std::vector< EntityHandle > dum_connect;
5167 std::vector< int > sp_vec;
5168
5169 for( int d = 3; d > 0; d-- )
5170 {
5171 if( resolve_dim == d ) continue;
5172
5173 for( Range::iterator rit = skin_ents[d].begin(); rit != skin_ents[d].end(); ++rit )
5174 {
5175
5176 result = mbImpl->get_connectivity( *rit, connect, num_connect, false, &dum_connect );MB_CHK_SET_ERR( result, "Failed to get connectivity on non-vertex skin entities" );
5177
5178 int op = ( resolve_dim < shared_dim ? Interface::UNION : Interface::INTERSECT );
5179 result = get_sharing_data( connect, num_connect, sharing_procs, op );MB_CHK_SET_ERR( result, "Failed to get sharing data in get_proc_nvecs" );
5180 if( sharing_procs.empty() ||
5181 ( sharing_procs.size() == 1 && *sharing_procs.begin() == (int)procConfig.proc_rank() ) )
5182 continue;
5183
5184
5185
5186 if( sharing_procs.size() == 2 )
5187 {
5188 std::set< int >::iterator it = sharing_procs.find( proc_config().proc_rank() );
5189 assert( it != sharing_procs.end() );
5190 sharing_procs.erase( it );
5191 }
5192
5193
5194 sp_vec.clear();
5195 std::copy( sharing_procs.begin(), sharing_procs.end(), std::back_inserter( sp_vec ) );
5196 assert( sp_vec.size() != 2 );
5197 proc_nvecs[sp_vec].push_back( *rit );
5198 }
5199 }
5200
5201 #ifndef NDEBUG
5202
5203 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
5204 mit != proc_nvecs.end(); ++mit )
5205 {
5206 std::vector< EntityHandle > tmp_vec = ( mit->second );
5207 std::sort( tmp_vec.begin(), tmp_vec.end() );
5208 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
5209 assert( vit == tmp_vec.end() );
5210 }
5211 #endif
5212
5213 return MB_SUCCESS;
5214 }
5215
5216
5217
5218
5219 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
5220 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
5221 Range& ,
5222 unsigned int i_extra )
5223 {
5224 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
5225 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
5226
5227 unsigned int j = 0, i = 0;
5228 std::vector< int > sharing_procs, sharing_procs2, tag_procs;
5229 std::vector< EntityHandle > sharing_handles, sharing_handles2, tag_lhandles, tag_rhandles;
5230 std::vector< unsigned char > pstatus;
5231
5232
5233 if( i_extra ) i += i_extra;
5234 while( j < 2 * shared_ents.get_n() )
5235 {
5236
5237 EntityHandle this_ent = shared_ents.vul_rd[j], other_ent = 0;
5238 int other_proc = -1;
5239 while( j < 2 * shared_ents.get_n() && shared_ents.vul_rd[j] == this_ent )
5240 {
5241 j++;
5242
5243 assert( shared_ents.vi_rd[i] != (int)procConfig.proc_rank() );
5244
5245 if( shared_ents.vul_rd[j] != other_ent || shared_ents.vi_rd[i] != other_proc )
5246 {
5247 assert( 0 != shared_ents.vul_rd[j] );
5248 sharing_procs.push_back( shared_ents.vi_rd[i] );
5249 sharing_handles.push_back( shared_ents.vul_rd[j] );
5250 }
5251 other_proc = shared_ents.vi_rd[i];
5252 other_ent = shared_ents.vul_rd[j];
5253 j++;
5254 i += 1 + i_extra;
5255 }
5256
5257 if( sharing_procs.size() > 1 )
5258 {
5259
5260 sharing_procs.push_back( procConfig.proc_rank() );
5261 sharing_handles.push_back( this_ent );
5262
5263
5264
5265
5266 sharing_procs2 = sharing_procs;
5267 std::sort( sharing_procs2.begin(), sharing_procs2.end() );
5268 sharing_handles2.resize( sharing_handles.size() );
5269 for( size_t k = 0; k < sharing_handles.size(); k++ )
5270 {
5271 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
5272 sharing_procs2.begin();
5273 sharing_handles2[idx] = sharing_handles[k];
5274 }
5275 sharing_procs.swap( sharing_procs2 );
5276 sharing_handles.swap( sharing_handles2 );
5277 }
5278
5279 assert( sharing_procs.size() != 2 );
5280 proc_nvecs[sharing_procs].push_back( this_ent );
5281
5282 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
5283 if( sharing_procs.size() == 1 )
5284 {
5285 tag_procs.push_back( sharing_procs[0] );
5286 tag_lhandles.push_back( this_ent );
5287 tag_rhandles.push_back( sharing_handles[0] );
5288 pstatus.push_back( share_flag );
5289 }
5290 else
5291 {
5292
5293
5294 if( sharing_procs.size() > MAX_SHARING_PROCS )
5295 {
5296 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
5297 << proc_config().proc_rank() << std::endl;
5298 std::cerr.flush();
5299 MPI_Abort( proc_config().proc_comm(), 66 );
5300 }
5301 sharing_procs.resize( MAX_SHARING_PROCS, -1 );
5302 sharing_handles.resize( MAX_SHARING_PROCS, 0 );
5303 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
5304 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
5305 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
5306 sharedEnts.insert( this_ent );
5307 }
5308
5309
5310 sharing_procs.clear();
5311 sharing_handles.clear();
5312 }
5313
5314 if( !tag_procs.empty() )
5315 {
5316 result = mbImpl->tag_set_data( shp_tag, &tag_lhandles[0], tag_procs.size(), &tag_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
5317 result = mbImpl->tag_set_data( shh_tag, &tag_lhandles[0], tag_procs.size(), &tag_rhandles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
5318 result = mbImpl->tag_set_data( pstat_tag, &tag_lhandles[0], tag_procs.size(), &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
5319 for( std::vector< EntityHandle >::iterator vvt = tag_lhandles.begin(); vvt != tag_lhandles.end(); vvt++ )
5320 sharedEnts.insert( *vvt );
5321 }
5322
5323 #ifndef NDEBUG
5324
5325 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
5326 mit != proc_nvecs.end(); ++mit )
5327 {
5328 std::vector< EntityHandle > tmp_vec = ( mit->second );
5329 std::sort( tmp_vec.begin(), tmp_vec.end() );
5330 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
5331 assert( vit == tmp_vec.end() );
5332 }
5333 #endif
5334
5335 return MB_SUCCESS;
5336 }
5337
5338 ErrorCode ParallelComm::tag_shared_verts( TupleList& shared_ents,
5339 Range* skin_ents,
5340 std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
5341 Range& )
5342 {
5343 Tag shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag;
5344 ErrorCode result = get_shared_proc_tags( shp_tag, shps_tag, shh_tag, shhs_tag, pstat_tag );MB_CHK_SET_ERR( result, "Failed to get shared proc tags in tag_shared_verts" );
5345
5346 unsigned int j = 0, i = 0;
5347 std::vector< int > sharing_procs, sharing_procs2;
5348 std::vector< EntityHandle > sharing_handles, sharing_handles2, skin_verts( skin_ents[0].size() );
5349 for( Range::iterator rit = skin_ents[0].begin(); rit != skin_ents[0].end(); ++rit, i++ )
5350 skin_verts[i] = *rit;
5351 i = 0;
5352
5353 while( j < 2 * shared_ents.get_n() )
5354 {
5355
5356 int this_idx = shared_ents.vi_rd[j];
5357 EntityHandle this_ent = skin_verts[this_idx];
5358 while( j < 2 * shared_ents.get_n() && shared_ents.vi_rd[j] == this_idx )
5359 {
5360 j++;
5361
5362 assert( shared_ents.vi_rd[j] != (int)procConfig.proc_rank() );
5363 sharing_procs.push_back( shared_ents.vi_rd[j++] );
5364 sharing_handles.push_back( shared_ents.vul_rd[i++] );
5365 }
5366
5367 if( sharing_procs.size() > 1 )
5368 {
5369
5370 sharing_procs.push_back( procConfig.proc_rank() );
5371 sharing_handles.push_back( this_ent );
5372 }
5373
5374
5375
5376
5377 sharing_procs2 = sharing_procs;
5378 std::sort( sharing_procs2.begin(), sharing_procs2.end() );
5379 sharing_handles2.resize( sharing_handles.size() );
5380 for( size_t k = 0; k < sharing_handles.size(); k++ )
5381 {
5382 size_t idx = std::lower_bound( sharing_procs2.begin(), sharing_procs2.end(), sharing_procs[k] ) -
5383 sharing_procs2.begin();
5384 sharing_handles2[idx] = sharing_handles[k];
5385 }
5386 sharing_procs.swap( sharing_procs2 );
5387 sharing_handles.swap( sharing_handles2 );
5388
5389 assert( sharing_procs.size() != 2 );
5390 proc_nvecs[sharing_procs].push_back( this_ent );
5391
5392 unsigned char share_flag = PSTATUS_SHARED, ms_flag = ( PSTATUS_SHARED | PSTATUS_MULTISHARED );
5393 if( sharing_procs.size() == 1 )
5394 {
5395 result = mbImpl->tag_set_data( shp_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedp tag on shared vertex" );
5396 result = mbImpl->tag_set_data( shh_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedh tag on shared vertex" );
5397 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &share_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
5398 sharedEnts.insert( this_ent );
5399 }
5400 else
5401 {
5402
5403
5404 if( sharing_procs.size() > MAX_SHARING_PROCS )
5405 {
5406 std::cerr << "MAX_SHARING_PROCS exceeded for vertex " << this_ent << " on process "
5407 << proc_config().proc_rank() << std::endl;
5408 std::cerr.flush();
5409 MPI_Abort( proc_config().proc_comm(), 66 );
5410 }
5411 sharing_procs.resize( MAX_SHARING_PROCS, -1 );
5412 sharing_handles.resize( MAX_SHARING_PROCS, 0 );
5413 result = mbImpl->tag_set_data( shps_tag, &this_ent, 1, &sharing_procs[0] );MB_CHK_SET_ERR( result, "Failed to set sharedps tag on shared vertex" );
5414 result = mbImpl->tag_set_data( shhs_tag, &this_ent, 1, &sharing_handles[0] );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag on shared vertex" );
5415 result = mbImpl->tag_set_data( pstat_tag, &this_ent, 1, &ms_flag );MB_CHK_SET_ERR( result, "Failed to set pstatus tag on shared vertex" );
5416 sharedEnts.insert( this_ent );
5417 }
5418
5419
5420 sharing_procs.clear();
5421 sharing_handles.clear();
5422 }
5423
5424 #ifndef NDEBUG
5425
5426 for( std::map< std::vector< int >, std::vector< EntityHandle > >::iterator mit = proc_nvecs.begin();
5427 mit != proc_nvecs.end(); ++mit )
5428 {
5429 std::vector< EntityHandle > tmp_vec = ( mit->second );
5430 std::sort( tmp_vec.begin(), tmp_vec.end() );
5431 std::vector< EntityHandle >::iterator vit = std::unique( tmp_vec.begin(), tmp_vec.end() );
5432 assert( vit == tmp_vec.end() );
5433 }
5434 #endif
5435
5436 return MB_SUCCESS;
5437 }
5438
5439
5440 ErrorCode ParallelComm::get_interface_procs( std::set< unsigned int >& procs_set, bool get_buffs )
5441 {
5442
5443 procs_set.clear();
5444
5445
5446 unsigned int i, j;
5447 std::vector< int > iface_proc( interfaceSets.size() );
5448 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), interfaceSets, &iface_proc[0] );MB_CHK_SET_ERR( result, "Failed to get iface_proc for iface sets" );
5449
5450
5451
5452 int tmp_iface_procs[MAX_SHARING_PROCS];
5453 std::fill( tmp_iface_procs, tmp_iface_procs + MAX_SHARING_PROCS, -1 );
5454 Range::iterator rit;
5455 for( rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); ++rit, i++ )
5456 {
5457 if( -1 != iface_proc[i] )
5458 {
5459 assert( iface_proc[i] != (int)procConfig.proc_rank() );
5460 procs_set.insert( (unsigned int)iface_proc[i] );
5461 }
5462 else
5463 {
5464
5465 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, tmp_iface_procs );MB_CHK_SET_ERR( result, "Failed to get iface_procs for iface set" );
5466 for( j = 0; j < MAX_SHARING_PROCS; j++ )
5467 {
5468 if( -1 != tmp_iface_procs[j] && tmp_iface_procs[j] != (int)procConfig.proc_rank() )
5469 procs_set.insert( (unsigned int)tmp_iface_procs[j] );
5470 else if( -1 == tmp_iface_procs[j] )
5471 {
5472 std::fill( tmp_iface_procs, tmp_iface_procs + j, -1 );
5473 break;
5474 }
5475 }
5476 }
5477 }
5478
5479 if( get_buffs )
5480 {
5481 for( std::set< unsigned int >::iterator sit = procs_set.begin(); sit != procs_set.end(); ++sit )
5482 get_buffers( *sit );
5483 }
5484
5485 return MB_SUCCESS;
5486 }
5487
5488 ErrorCode ParallelComm::get_pstatus( EntityHandle entity, unsigned char& pstatus_val )
5489 {
5490 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstatus_val );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
5491 return result;
5492 }
5493
5494 ErrorCode ParallelComm::get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents )
5495 {
5496 Range ents;
5497 ErrorCode result;
5498
5499 if( -1 == dim )
5500 {
5501 result = mbImpl->get_entities_by_handle( 0, ents );MB_CHK_SET_ERR( result, "Failed to get all entities" );
5502 }
5503 else
5504 {
5505 result = mbImpl->get_entities_by_dimension( 0, dim, ents );MB_CHK_SET_ERR( result, "Failed to get entities of dimension " << dim );
5506 }
5507
5508 std::vector< unsigned char > pstatus( ents.size() );
5509 result = mbImpl->tag_get_data( pstatus_tag(), ents, &pstatus[0] );MB_CHK_SET_ERR( result, "Failed to get pastatus tag data" );
5510 Range::iterator rit = ents.begin();
5511 int i = 0;
5512 if( pstatus_val )
5513 {
5514 for( ; rit != ents.end(); i++, ++rit )
5515 {
5516 if( pstatus[i] & pstatus_val && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
5517 pstatus_ents.insert( *rit );
5518 }
5519 }
5520 else
5521 {
5522 for( ; rit != ents.end(); i++, ++rit )
5523 {
5524 if( !pstatus[i] && ( -1 == dim || mbImpl->dimension_from_handle( *rit ) == dim ) )
5525 pstatus_ents.insert( *rit );
5526 }
5527 }
5528
5529 return MB_SUCCESS;
5530 }
5531
5532 ErrorCode ParallelComm::check_global_ids( EntityHandle this_set,
5533 const int dimension,
5534 const int start_id,
5535 const bool largest_dim_only,
5536 const bool parallel,
5537 const bool owned_only )
5538 {
5539
5540 Tag gid_tag = mbImpl->globalId_tag();
5541 int def_val = -1;
5542 Range dum_range;
5543
5544 void* tag_ptr = &def_val;
5545 ErrorCode result = mbImpl->get_entities_by_type_and_tag( this_set, MBVERTEX, &gid_tag, &tag_ptr, 1, dum_range );MB_CHK_SET_ERR( result, "Failed to get entities by MBVERTEX type and gid tag" );
5546
5547 if( !dum_range.empty() )
5548 {
5549
5550 result = assign_global_ids( this_set, dimension, start_id, largest_dim_only, parallel, owned_only );MB_CHK_SET_ERR( result, "Failed assigning global ids" );
5551 }
5552
5553 return MB_SUCCESS;
5554 }
5555
5556 bool ParallelComm::is_iface_proc( EntityHandle this_set, int to_proc )
5557 {
5558 int sharing_procs[MAX_SHARING_PROCS];
5559 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
5560 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &this_set, 1, sharing_procs );
5561 if( MB_SUCCESS == result && to_proc == sharing_procs[0] ) return true;
5562
5563 result = mbImpl->tag_get_data( sharedps_tag(), &this_set, 1, sharing_procs );
5564 if( MB_SUCCESS != result ) return false;
5565
5566 for( int i = 0; i < MAX_SHARING_PROCS; i++ )
5567 {
5568 if( to_proc == sharing_procs[i] )
5569 return true;
5570 else if( -1 == sharing_procs[i] )
5571 return false;
5572 }
5573
5574 return false;
5575 }
5576
5577 ErrorCode ParallelComm::filter_pstatus( Range& ents,
5578 unsigned char pstat,
5579 unsigned char op,
5580 int to_proc,
5581 Range* returned_ents )
5582 {
5583 Range tmp_ents;
5584
5585
5586 if( ents.empty() )
5587 {
5588 if( returned_ents ) returned_ents->clear();
5589 return MB_SUCCESS;
5590 }
5591
5592
5593
5594 std::vector< unsigned char > shared_flags( ents.size() ), shared_flags2;
5595 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), ents, &shared_flags[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus flag" );
5596 Range::const_iterator rit, hint = tmp_ents.begin();
5597 ;
5598 int i;
5599 if( op == PSTATUS_OR )
5600 {
5601 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
5602 {
5603 if( ( ( shared_flags[i] & ~pstat ) ^ shared_flags[i] ) & pstat )
5604 {
5605 hint = tmp_ents.insert( hint, *rit );
5606 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
5607 }
5608 }
5609 }
5610 else if( op == PSTATUS_AND )
5611 {
5612 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
5613 {
5614 if( ( shared_flags[i] & pstat ) == pstat )
5615 {
5616 hint = tmp_ents.insert( hint, *rit );
5617 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
5618 }
5619 }
5620 }
5621 else if( op == PSTATUS_NOT )
5622 {
5623 for( rit = ents.begin(), i = 0; rit != ents.end(); ++rit, i++ )
5624 {
5625 if( !( shared_flags[i] & pstat ) )
5626 {
5627 hint = tmp_ents.insert( hint, *rit );
5628 if( -1 != to_proc ) shared_flags2.push_back( shared_flags[i] );
5629 }
5630 }
5631 }
5632 else
5633 {
5634 assert( false );
5635 return MB_FAILURE;
5636 }
5637
5638 if( -1 != to_proc )
5639 {
5640 int sharing_procs[MAX_SHARING_PROCS];
5641 std::fill( sharing_procs, sharing_procs + MAX_SHARING_PROCS, -1 );
5642 Range tmp_ents2;
5643 hint = tmp_ents2.begin();
5644
5645 for( rit = tmp_ents.begin(), i = 0; rit != tmp_ents.end(); ++rit, i++ )
5646 {
5647
5648 if( shared_flags2[i] & PSTATUS_MULTISHARED )
5649 {
5650 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag" );
5651 assert( -1 != sharing_procs[0] );
5652 for( unsigned int j = 0; j < MAX_SHARING_PROCS; j++ )
5653 {
5654
5655 if( sharing_procs[j] == to_proc )
5656 {
5657 hint = tmp_ents2.insert( hint, *rit );
5658 }
5659 else if( -1 == sharing_procs[j] )
5660 break;
5661
5662 sharing_procs[j] = -1;
5663 }
5664 }
5665 else if( shared_flags2[i] & PSTATUS_SHARED )
5666 {
5667 result = mbImpl->tag_get_data( sharedp_tag(), &( *rit ), 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag" );
5668 assert( -1 != sharing_procs[0] );
5669 if( sharing_procs[0] == to_proc ) hint = tmp_ents2.insert( hint, *rit );
5670 sharing_procs[0] = -1;
5671 }
5672 else
5673 assert( "should never get here" && false );
5674 }
5675
5676 tmp_ents.swap( tmp_ents2 );
5677 }
5678
5679 if( returned_ents )
5680 returned_ents->swap( tmp_ents );
5681 else
5682 ents.swap( tmp_ents );
5683
5684 return MB_SUCCESS;
5685 }
5686
5687 ErrorCode ParallelComm::exchange_ghost_cells( int ghost_dim,
5688 int bridge_dim,
5689 int num_layers,
5690 int addl_ents,
5691 bool store_remote_handles,
5692 bool wait_all,
5693 EntityHandle* file_set )
5694 {
5695 #ifdef MOAB_HAVE_MPE
5696 if( myDebug->get_verbosity() == 2 )
5697 {
5698 if( !num_layers )
5699 MPE_Log_event( IFACE_START, procConfig.proc_rank(), "Starting interface exchange." );
5700 else
5701 MPE_Log_event( GHOST_START, procConfig.proc_rank(), "Starting ghost exchange." );
5702 }
5703 #endif
5704
5705 myDebug->tprintf( 1, "Entering exchange_ghost_cells with num_layers = %d\n", num_layers );
5706 if( myDebug->get_verbosity() == 4 )
5707 {
5708 msgs.clear();
5709 msgs.reserve( MAX_SHARING_PROCS );
5710 }
5711
5712
5713
5714 assert( num_layers > 0 || store_remote_handles );
5715
5716 const bool is_iface = !num_layers;
5717
5718
5719
5720 int success;
5721 ErrorCode result = MB_SUCCESS;
5722 int incoming1 = 0, incoming2 = 0;
5723
5724 reset_all_buffers();
5725
5726
5727
5728
5729
5730
5731
5732 #ifdef MOAB_HAVE_MPE
5733 if( myDebug->get_verbosity() == 2 )
5734 {
5735 MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." );
5736 }
5737 #endif
5738
5739
5740 std::vector< MPI_Request > recv_ent_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL ),
5741 recv_remoteh_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
5742 std::vector< unsigned int >::iterator proc_it;
5743 int ind, p;
5744 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
5745 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
5746 {
5747 incoming1++;
5748 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
5749 MB_MESG_ENTS_SIZE, incoming1 );
5750 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
5751 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
5752 if( success != MPI_SUCCESS )
5753 {
5754 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
5755 }
5756 }
5757
5758
5759
5760
5761 Range sent_ents[MAX_SHARING_PROCS], allsent, tmp_range;
5762 TupleList entprocs;
5763 int dum_ack_buff;
5764 result = get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents, allsent, entprocs );MB_CHK_SET_ERR( result, "get_sent_ents failed" );
5765
5766
5767
5768 if( addl_ents > 0 && file_set && !allsent.empty() )
5769 {
5770 result = mbImpl->add_entities( *file_set, allsent );MB_CHK_SET_ERR( result, "Failed to add new sub-entities to set" );
5771 }
5772 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
5773 (unsigned long)allsent.size() );
5774
5775
5776
5777
5778 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
5779 {
5780 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", sent_ents[p].compactness(),
5781 (unsigned long)sent_ents[p].size() );
5782
5783
5784 localOwnedBuffs[p]->reset_buffer( sizeof( int ) );
5785
5786
5787 result = pack_entities( sent_ents[p], localOwnedBuffs[p], store_remote_handles, buffProcs[p], is_iface,
5788 &entprocs, &allsent );MB_CHK_SET_ERR( result, "Packing entities failed" );
5789
5790 if( myDebug->get_verbosity() == 4 )
5791 {
5792 msgs.resize( msgs.size() + 1 );
5793 msgs.back() = new Buffer( *localOwnedBuffs[p] );
5794 }
5795
5796
5797 result = send_buffer( *proc_it, localOwnedBuffs[p], MB_MESG_ENTS_SIZE, sendReqs[3 * p],
5798 recv_ent_reqs[3 * p + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE,
5799 ( !is_iface && store_remote_handles ?
5800 localOwnedBuffs[p]
5801 : NULL ),
5802 &recv_remoteh_reqs[3 * p], &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" );
5803 }
5804
5805 entprocs.reset();
5806
5807
5808
5809
5810
5811
5812 MPI_Status status;
5813 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
5814 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
5815 std::vector< std::vector< int > > L1p( buffProcs.size() );
5816 std::vector< EntityHandle > L2hloc, L2hrem;
5817 std::vector< unsigned int > L2p;
5818 std::vector< EntityHandle > new_ents;
5819
5820 while( incoming1 )
5821 {
5822
5823
5824 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
5825
5826 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status );
5827 if( MPI_SUCCESS != success )
5828 {
5829 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
5830 }
5831
5832 PRINT_DEBUG_RECD( status );
5833
5834
5835 incoming1--;
5836 bool done = false;
5837
5838
5839 unsigned int base_ind = 3 * ( ind / 3 );
5840 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1],
5841 recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1],
5842 sendReqs[base_ind + 2], done,
5843 ( !is_iface && store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ),
5844 MB_MESG_REMOTEH_SIZE,
5845 &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
5846
5847 if( done )
5848 {
5849 if( myDebug->get_verbosity() == 4 )
5850 {
5851 msgs.resize( msgs.size() + 1 );
5852 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] );
5853 }
5854
5855
5856 remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
5857 result = unpack_entities( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, ind / 3, is_iface,
5858 L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents );
5859 if( MB_SUCCESS != result )
5860 {
5861 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
5862 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false );
5863 return result;
5864 }
5865
5866 if( recv_ent_reqs.size() != 3 * buffProcs.size() )
5867 {
5868
5869
5870 assert( !is_iface );
5871 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
5872 for( unsigned int i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 )
5873 {
5874 localOwnedBuffs[i / 3]->reset_buffer();
5875 incoming2++;
5876 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr,
5877 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
5878 success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
5879 buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
5880 &recv_remoteh_reqs[i] );
5881 if( success != MPI_SUCCESS )
5882 {
5883 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
5884 }
5885 }
5886 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
5887 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
5888 }
5889 }
5890 }
5891
5892
5893 if( recv_ent_reqs.size() != 3 * buffProcs.size() )
5894 {
5895
5896 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in ghost exchange" );
5897 }
5898
5899 #ifdef MOAB_HAVE_MPE
5900 if( myDebug->get_verbosity() == 2 )
5901 {
5902 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." );
5903 }
5904 #endif
5905
5906 if( is_iface )
5907 {
5908
5909
5910
5911 result = check_clean_iface( allsent );
5912 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
5913
5914
5915 result = tag_iface_entities();MB_CHK_SET_ERR( result, "Failed to tag iface entities" );
5916
5917 #ifndef NDEBUG
5918 result = check_sent_ents( allsent );
5919 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
5920 result = check_all_shared_handles( true );
5921 if( MB_SUCCESS != result ) std::cout << "Failed check." << std::endl;
5922 #endif
5923
5924 #ifdef MOAB_HAVE_MPE
5925 if( myDebug->get_verbosity() == 2 )
5926 {
5927 MPE_Log_event( IFACE_END, procConfig.proc_rank(), "Ending interface exchange." );
5928 }
5929 #endif
5930
5931
5932
5933
5934 if( wait_all )
5935 {
5936 if( myDebug->get_verbosity() == 5 )
5937 {
5938 success = MPI_Barrier( procConfig.proc_comm() );
5939 }
5940 else
5941 {
5942 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
5943 success = MPI_Waitall( 3 * buffProcs.size(), &recv_ent_reqs[0], mult_status );
5944 if( MPI_SUCCESS != success )
5945 {
5946 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
5947 }
5948 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
5949 if( MPI_SUCCESS != success )
5950 {
5951 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
5952 }
5953
5957 }
5958 }
5959
5960 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
5961 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==true \n" );
5962
5963 return MB_SUCCESS;
5964 }
5965
5966
5967 if( wait_all )
5968 {
5969 if( myDebug->get_verbosity() == 5 )
5970 {
5971 success = MPI_Barrier( procConfig.proc_comm() );
5972 }
5973 else
5974 {
5975 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
5976 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
5977 if( MPI_SUCCESS != success )
5978 {
5979 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
5980 }
5981 }
5982 }
5983
5984
5985
5986
5987 for( p = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, p++ )
5988 {
5989
5990
5991 remoteOwnedBuffs[p]->reset_buffer( sizeof( int ) );
5992
5993 result = pack_remote_handles( L1hloc[p], L1hrem[p], L1p[p], *proc_it, remoteOwnedBuffs[p] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
5994 remoteOwnedBuffs[p]->set_stored_size();
5995
5996 if( myDebug->get_verbosity() == 4 )
5997 {
5998 msgs.resize( msgs.size() + 1 );
5999 msgs.back() = new Buffer( *remoteOwnedBuffs[p] );
6000 }
6001 result = send_buffer( buffProcs[p], remoteOwnedBuffs[p], MB_MESG_REMOTEH_SIZE, sendReqs[3 * p],
6002 recv_remoteh_reqs[3 * p + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
6003 }
6004
6005
6006
6007
6008 while( incoming2 )
6009 {
6010 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
6011 success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status );
6012 if( MPI_SUCCESS != success )
6013 {
6014 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
6015 }
6016
6017
6018 incoming2--;
6019
6020 PRINT_DEBUG_RECD( status );
6021
6022 bool done = false;
6023 unsigned int base_ind = 3 * ( ind / 3 );
6024 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1],
6025 recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3],
6026 sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
6027 if( done )
6028 {
6029
6030 if( myDebug->get_verbosity() == 4 )
6031 {
6032 msgs.resize( msgs.size() + 1 );
6033 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] );
6034 }
6035 localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
6036 result =
6037 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
6038 }
6039 }
6040
6041 #ifdef MOAB_HAVE_MPE
6042 if( myDebug->get_verbosity() == 2 )
6043 {
6044 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
6045 MPE_Log_event( GHOST_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." );
6046 }
6047 #endif
6048
6049
6050
6051
6052 if( wait_all )
6053 {
6054 if( myDebug->get_verbosity() == 5 )
6055 {
6056 success = MPI_Barrier( procConfig.proc_comm() );
6057 }
6058 else
6059 {
6060 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
6061 success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status );
6062 if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
6063 }
6064 if( MPI_SUCCESS != success )
6065 {
6066 MB_SET_ERR( MB_FAILURE, "Failed in waitall in ghost exchange" );
6067 }
6068 }
6069
6070 #ifndef NDEBUG
6071 result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" );
6072 result = check_all_shared_handles( true );MB_CHK_SET_ERR( result, "Failed check on all shared handles" );
6073 #endif
6074
6075 if( file_set && !new_ents.empty() )
6076 {
6077 result = mbImpl->add_entities( *file_set, &new_ents[0], new_ents.size() );MB_CHK_SET_ERR( result, "Failed to add new entities to set" );
6078 }
6079
6080 myDebug->tprintf( 1, "Total number of shared entities = %lu.\n", (unsigned long)sharedEnts.size() );
6081 myDebug->tprintf( 1, "Exiting exchange_ghost_cells for is_iface==false \n" );
6082
6083 return MB_SUCCESS;
6084 }
6085
6086 ErrorCode ParallelComm::send_buffer( const unsigned int to_proc,
6087 Buffer* send_buff,
6088 int mesg_tag,
6089 MPI_Request& send_req,
6090 MPI_Request& ack_req,
6091 int* ack_buff,
6092 int& this_incoming,
6093 int next_mesg_tag,
6094 Buffer* next_recv_buff,
6095 MPI_Request* next_recv_req,
6096 int* next_incoming )
6097 {
6098 ErrorCode result = MB_SUCCESS;
6099 int success;
6100
6101
6102 if( send_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE && next_recv_buff )
6103 {
6104 ( *next_incoming )++;
6105 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, next_mesg_tag,
6106 *next_incoming );
6107 success = MPI_Irecv( next_recv_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, to_proc, next_mesg_tag,
6108 procConfig.proc_comm(), next_recv_req );
6109 if( success != MPI_SUCCESS )
6110 {
6111 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for next message in ghost exchange" );
6112 }
6113 }
6114
6115 else if( send_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
6116 {
6117 this_incoming++;
6118 PRINT_DEBUG_IRECV( procConfig.proc_rank(), to_proc, (unsigned char*)ack_buff, sizeof( int ), mesg_tag - 1,
6119 this_incoming );
6120 success = MPI_Irecv( (void*)ack_buff, sizeof( int ), MPI_UNSIGNED_CHAR, to_proc, mesg_tag - 1,
6121 procConfig.proc_comm(), &ack_req );
6122 if( success != MPI_SUCCESS )
6123 {
6124 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for entity ack in ghost exchange" );
6125 }
6126 }
6127
6128
6129 PRINT_DEBUG_ISEND( procConfig.proc_rank(), to_proc, send_buff->mem_ptr, mesg_tag,
6130 std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ) );
6131 assert( 0 <= send_buff->get_stored_size() && send_buff->get_stored_size() <= (int)send_buff->alloc_size );
6132 success = MPI_Isend( send_buff->mem_ptr, std::min( send_buff->get_stored_size(), (int)INITIAL_BUFF_SIZE ),
6133 MPI_UNSIGNED_CHAR, to_proc, mesg_tag, procConfig.proc_comm(), &send_req );
6134 if( success != MPI_SUCCESS ) return MB_FAILURE;
6135
6136 return result;
6137 }
6138
6139 ErrorCode ParallelComm::recv_buffer( int mesg_tag_expected,
6140 const MPI_Status& mpi_status,
6141 Buffer* recv_buff,
6142 MPI_Request& recv_req,
6143 MPI_Request& ,
6144 int& this_incoming,
6145 Buffer* send_buff,
6146 MPI_Request& send_req,
6147 MPI_Request& sent_ack_req,
6148 bool& done,
6149 Buffer* next_buff,
6150 int next_tag,
6151 MPI_Request* next_req,
6152 int* next_incoming )
6153 {
6154
6155
6156 int from_proc = mpi_status.MPI_SOURCE;
6157 int success;
6158
6159
6160
6161 recv_buff->reset_ptr( std::min( recv_buff->get_stored_size(), (int)recv_buff->alloc_size ) );
6162
6163 if( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() > (int)INITIAL_BUFF_SIZE )
6164 {
6165
6166
6167 recv_buff->reserve( recv_buff->get_stored_size() );
6168 assert( recv_buff->alloc_size > INITIAL_BUFF_SIZE );
6169
6170
6171 this_incoming++;
6172
6173 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr + INITIAL_BUFF_SIZE,
6174 recv_buff->get_stored_size() - INITIAL_BUFF_SIZE, mesg_tag_expected + 1, this_incoming );
6175 success = MPI_Irecv( recv_buff->mem_ptr + INITIAL_BUFF_SIZE, recv_buff->get_stored_size() - INITIAL_BUFF_SIZE,
6176 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &recv_req );
6177 if( success != MPI_SUCCESS )
6178 {
6179 MB_SET_ERR( MB_FAILURE, "Failed to post 2nd iRecv in ghost exchange" );
6180 }
6181
6182
6183 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, recv_buff->mem_ptr, mesg_tag_expected - 1,
6184 sizeof( int ) );
6185 success = MPI_Isend( recv_buff->mem_ptr, sizeof( int ), MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected - 1,
6186 procConfig.proc_comm(), &sent_ack_req );
6187 if( success != MPI_SUCCESS )
6188 {
6189 MB_SET_ERR( MB_FAILURE, "Failed to send ack in ghost exchange" );
6190 }
6191 }
6192 else if( mpi_status.MPI_TAG == mesg_tag_expected - 1 )
6193 {
6194
6195
6196
6197 assert( *( (size_t*)send_buff->mem_ptr ) > INITIAL_BUFF_SIZE );
6198
6199
6200 if( next_buff )
6201 {
6202
6203 ( *next_incoming )++;
6204 PRINT_DEBUG_IRECV( procConfig.proc_rank(), from_proc, next_buff->mem_ptr, INITIAL_BUFF_SIZE, next_tag,
6205 *next_incoming );
6206
6207 success = MPI_Irecv( next_buff->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, from_proc, next_tag,
6208 procConfig.proc_comm(), next_req );
6209 if( success != MPI_SUCCESS )
6210 {
6211 MB_SET_ERR( MB_FAILURE, "Failed to post next irecv in ghost exchange" );
6212 }
6213 }
6214
6215
6216 PRINT_DEBUG_ISEND( procConfig.proc_rank(), from_proc, send_buff->mem_ptr + INITIAL_BUFF_SIZE,
6217 mesg_tag_expected + 1, send_buff->get_stored_size() - INITIAL_BUFF_SIZE );
6218
6219 assert( send_buff->get_stored_size() - INITIAL_BUFF_SIZE < send_buff->alloc_size &&
6220 0 <= send_buff->get_stored_size() );
6221 success = MPI_Isend( send_buff->mem_ptr + INITIAL_BUFF_SIZE, send_buff->get_stored_size() - INITIAL_BUFF_SIZE,
6222 MPI_UNSIGNED_CHAR, from_proc, mesg_tag_expected + 1, procConfig.proc_comm(), &send_req );
6223 if( success != MPI_SUCCESS )
6224 {
6225 MB_SET_ERR( MB_FAILURE, "Failed to send 2nd message in ghost exchange" );
6226 }
6227 }
6228 else if( ( mpi_status.MPI_TAG == mesg_tag_expected && recv_buff->get_stored_size() <= (int)INITIAL_BUFF_SIZE ) ||
6229 mpi_status.MPI_TAG == mesg_tag_expected + 1 )
6230 {
6231
6232 done = true;
6233 }
6234
6235 return MB_SUCCESS;
6236 }
6237
6238 struct ProcList
6239 {
6240 int procs[MAX_SHARING_PROCS];
6241 };
6242 static bool operator<( const ProcList& a, const ProcList& b )
6243 {
6244 for( int i = 0; i < MAX_SHARING_PROCS; i++ )
6245 {
6246 if( a.procs[i] < b.procs[i] )
6247 return true;
6248 else if( b.procs[i] < a.procs[i] )
6249 return false;
6250 else if( a.procs[i] < 0 )
6251 return false;
6252 }
6253 return false;
6254 }
6255
6256 ErrorCode ParallelComm::check_clean_iface( Range& allsent )
6257 {
6258
6259
6260
6261
6262
6263 typedef std::map< ProcList, Range > procmap_t;
6264 procmap_t old_procs, new_procs;
6265
6266 ErrorCode result = MB_SUCCESS;
6267 Range::iterator rit;
6268 Range::reverse_iterator rvit;
6269 unsigned char pstatus;
6270 int nump;
6271 ProcList sharedp;
6272 EntityHandle sharedh[MAX_SHARING_PROCS];
6273 for( rvit = allsent.rbegin(); rvit != allsent.rend(); ++rvit )
6274 {
6275 result = get_sharing_data( *rvit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data" );
6276 assert( "Should be shared with at least one other proc" &&
6277 ( nump > 1 || sharedp.procs[0] != (int)procConfig.proc_rank() ) );
6278 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
6279
6280
6281 int idx = std::find( sharedh, sharedh + nump, (EntityHandle)0 ) - sharedh;
6282 if( idx == nump ) continue;
6283
6284 ProcList old_list( sharedp );
6285 std::sort( old_list.procs, old_list.procs + nump );
6286 old_procs[old_list].insert( *rvit );
6287
6288
6289 int new_nump = idx;
6290 bool removed_owner = !idx;
6291 for( ++idx; idx < nump; ++idx )
6292 {
6293 if( sharedh[idx] )
6294 {
6295 sharedh[new_nump] = sharedh[idx];
6296 sharedp.procs[new_nump] = sharedp.procs[idx];
6297 ++new_nump;
6298 }
6299 }
6300 sharedp.procs[new_nump] = -1;
6301
6302 if( removed_owner && new_nump > 1 )
6303 {
6304
6305
6306
6307 idx = std::min_element( sharedp.procs, sharedp.procs + new_nump ) - sharedp.procs;
6308 std::swap( sharedp.procs[0], sharedp.procs[idx] );
6309 std::swap( sharedh[0], sharedh[idx] );
6310 if( sharedp.procs[0] == (int)proc_config().proc_rank() ) pstatus &= ~PSTATUS_NOT_OWNED;
6311 }
6312
6313 result = set_sharing_data( *rvit, pstatus, nump, new_nump, sharedp.procs, sharedh );MB_CHK_SET_ERR( result, "Failed to set sharing data in check_clean_iface" );
6314
6315 if( new_nump > 1 )
6316 {
6317 if( new_nump == 2 )
6318 {
6319 if( sharedp.procs[1] != (int)proc_config().proc_rank() )
6320 {
6321 assert( sharedp.procs[0] == (int)proc_config().proc_rank() );
6322 sharedp.procs[0] = sharedp.procs[1];
6323 }
6324 sharedp.procs[1] = -1;
6325 }
6326 else
6327 {
6328 std::sort( sharedp.procs, sharedp.procs + new_nump );
6329 }
6330 new_procs[sharedp].insert( *rvit );
6331 }
6332 }
6333
6334 if( old_procs.empty() )
6335 {
6336 assert( new_procs.empty() );
6337 return MB_SUCCESS;
6338 }
6339
6340
6341 procmap_t::iterator pmit;
6342
6343 rit = interface_sets().begin();
6344 while( rit != interface_sets().end() )
6345 {
6346 result = get_sharing_data( *rit, sharedp.procs, sharedh, pstatus, nump );MB_CHK_SET_ERR( result, "Failed to get sharing data for interface set" );
6347 assert( nump != 2 );
6348 std::sort( sharedp.procs, sharedp.procs + nump );
6349 assert( nump == MAX_SHARING_PROCS || sharedp.procs[nump] == -1 );
6350
6351 pmit = old_procs.find( sharedp );
6352 if( pmit != old_procs.end() )
6353 {
6354 result = mbImpl->remove_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to remove entities from interface set" );
6355 }
6356
6357 pmit = new_procs.find( sharedp );
6358 if( pmit == new_procs.end() )
6359 {
6360 int count;
6361 result = mbImpl->get_number_entities_by_handle( *rit, count );MB_CHK_SET_ERR( result, "Failed to get number of entities in interface set" );
6362 if( !count )
6363 {
6364 result = mbImpl->delete_entities( &*rit, 1 );MB_CHK_SET_ERR( result, "Failed to delete entities from interface set" );
6365 rit = interface_sets().erase( rit );
6366 }
6367 else
6368 {
6369 ++rit;
6370 }
6371 }
6372 else
6373 {
6374 result = mbImpl->add_entities( *rit, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
6375
6376
6377
6378 new_procs.erase( pmit );
6379 ++rit;
6380 }
6381 }
6382
6383
6384 std::fill( sharedh, sharedh + MAX_SHARING_PROCS, 0 );
6385 for( pmit = new_procs.begin(); pmit != new_procs.end(); ++pmit )
6386 {
6387 EntityHandle new_set;
6388 result = mbImpl->create_meshset( MESHSET_SET, new_set );MB_CHK_SET_ERR( result, "Failed to create interface set" );
6389 interfaceSets.insert( new_set );
6390
6391
6392 result = mbImpl->add_entities( new_set, pmit->second );MB_CHK_SET_ERR( result, "Failed to add entities to interface set" );
6393
6394 assert( pmit->first.procs[0] >= 0 );
6395 pstatus = PSTATUS_SHARED | PSTATUS_INTERFACE;
6396 if( pmit->first.procs[1] == -1 )
6397 {
6398 int other = pmit->first.procs[0];
6399 assert( other != (int)procConfig.proc_rank() );
6400 result = mbImpl->tag_set_data( sharedp_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
6401 sharedh[0] = 0;
6402 result = mbImpl->tag_set_data( sharedh_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
6403 if( other < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
6404 }
6405 else
6406 {
6407 result = mbImpl->tag_set_data( sharedps_tag(), &new_set, 1, pmit->first.procs );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
6408 result = mbImpl->tag_set_data( sharedhs_tag(), &new_set, 1, sharedh );MB_CHK_SET_ERR( result, "Failed to tag interface set with procs" );
6409 pstatus |= PSTATUS_MULTISHARED;
6410 if( pmit->first.procs[0] < (int)proc_config().proc_rank() ) pstatus |= PSTATUS_NOT_OWNED;
6411 }
6412
6413 result = mbImpl->tag_set_data( pstatus_tag(), &new_set, 1, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface set with pstatus" );
6414
6415
6416 result = mbImpl->tag_clear_data( pstatus_tag(), pmit->second, &pstatus );MB_CHK_SET_ERR( result, "Failed to tag interface entities with pstatus" );
6417 }
6418
6419 return MB_SUCCESS;
6420 }
6421
6422 ErrorCode ParallelComm::set_sharing_data( EntityHandle ent,
6423 unsigned char pstatus,
6424 int old_nump,
6425 int new_nump,
6426 int* ps,
6427 EntityHandle* hs )
6428 {
6429
6430 if( old_nump > 2 && ( pstatus & PSTATUS_MULTISHARED ) && new_nump < 3 )
6431 {
6432
6433 pstatus ^= PSTATUS_MULTISHARED;
6434 }
6435
6436
6437
6438
6445
6446
6447
6448 assert( new_nump > 1 &&
6449 ( ( new_nump == 2 && pstatus & PSTATUS_SHARED &&
6450 !( pstatus & PSTATUS_MULTISHARED ) ) ||
6451 ( new_nump > 2 && pstatus & PSTATUS_SHARED &&
6452 pstatus & PSTATUS_MULTISHARED ) ) &&
6453 ( !( pstatus & PSTATUS_GHOST ) || pstatus & PSTATUS_SHARED ) &&
6454 ( new_nump < 3 ||
6455 ( pstatus & PSTATUS_NOT_OWNED && ps[0] != (int)rank() ) ||
6456 ( !( pstatus & PSTATUS_NOT_OWNED ) && ps[0] == (int)rank() ) )
6457 );
6458
6459 #ifndef NDEBUG
6460 {
6461
6462 std::set< unsigned int > dumprocs;
6463 int dp = 0;
6464 for( ; dp < old_nump && -1 != ps[dp]; dp++ )
6465 dumprocs.insert( ps[dp] );
6466 assert( dp == (int)dumprocs.size() );
6467 }
6468 #endif
6469
6470 ErrorCode result;
6471
6472 if( old_nump > 2 && new_nump < 3 )
6473 {
6474
6475 result = mbImpl->tag_delete_data( sharedps_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:1" );
6476 result = mbImpl->tag_delete_data( sharedhs_tag(), &ent, 1 );MB_CHK_SET_ERR( result, "set_sharing_data:2" );
6477
6478
6479
6480
6481 }
6482 else if( ( old_nump < 3 && new_nump > 2 ) || ( old_nump > 1 && new_nump == 1 ) )
6483 {
6484
6485 int tmp_p = -1;
6486 EntityHandle tmp_h = 0;
6487 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, &tmp_p );MB_CHK_SET_ERR( result, "set_sharing_data:3" );
6488 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, &tmp_h );MB_CHK_SET_ERR( result, "set_sharing_data:4" );
6489 }
6490
6491 assert( "check for multishared/owner I'm first proc" &&
6492 ( !( pstatus & PSTATUS_MULTISHARED ) || ( pstatus & ( PSTATUS_NOT_OWNED | PSTATUS_GHOST ) ) ||
6493 ( ps[0] == (int)rank() ) ) &&
6494 "interface entities should have > 1 proc" && ( !( pstatus & PSTATUS_INTERFACE ) || new_nump > 1 ) &&
6495 "ghost entities should have > 1 proc" && ( !( pstatus & PSTATUS_GHOST ) || new_nump > 1 ) );
6496
6497
6498 if( new_nump > 2 )
6499 {
6500 result = mbImpl->tag_set_data( sharedps_tag(), &ent, 1, ps );MB_CHK_SET_ERR( result, "set_sharing_data:5" );
6501 result = mbImpl->tag_set_data( sharedhs_tag(), &ent, 1, hs );MB_CHK_SET_ERR( result, "set_sharing_data:6" );
6502 }
6503 else
6504 {
6505 unsigned int j = ( ps[0] == (int)procConfig.proc_rank() ? 1 : 0 );
6506 assert( -1 != ps[j] );
6507 result = mbImpl->tag_set_data( sharedp_tag(), &ent, 1, ps + j );MB_CHK_SET_ERR( result, "set_sharing_data:7" );
6508 result = mbImpl->tag_set_data( sharedh_tag(), &ent, 1, hs + j );MB_CHK_SET_ERR( result, "set_sharing_data:8" );
6509 }
6510
6511 result = mbImpl->tag_set_data( pstatus_tag(), &ent, 1, &pstatus );MB_CHK_SET_ERR( result, "set_sharing_data:9" );
6512
6513 if( old_nump > 1 && new_nump < 2 ) sharedEnts.erase( ent );
6514
6515 return result;
6516 }
6517
6518 ErrorCode ParallelComm::get_sent_ents( const bool is_iface,
6519 const int bridge_dim,
6520 const int ghost_dim,
6521 const int num_layers,
6522 const int addl_ents,
6523 Range* sent_ents,
6524 Range& allsent,
6525 TupleList& entprocs )
6526 {
6527 ErrorCode result;
6528 unsigned int ind;
6529 std::vector< unsigned int >::iterator proc_it;
6530 Range tmp_range;
6531
6532
6533
6534 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
6535 {
6536 if( !is_iface )
6537 {
6538 result =
6539 get_ghosted_entities( bridge_dim, ghost_dim, buffProcs[ind], num_layers, addl_ents, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get ghost layers" );
6540 }
6541 else
6542 {
6543 result = get_iface_entities( buffProcs[ind], -1, sent_ents[ind] );MB_CHK_SET_ERR( result, "Failed to get interface layers" );
6544 }
6545
6546
6547 tmp_range.clear();
6548 result = filter_pstatus( sent_ents[ind], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
6549 if( !tmp_range.empty() ) sent_ents[ind] = subtract( sent_ents[ind], tmp_range );
6550
6551 allsent.merge( sent_ents[ind] );
6552 }
6553
6554
6555
6556
6557
6558
6559 int npairs = 0;
6560 for( ind = 0; ind < buffProcs.size(); ind++ )
6561 npairs += sent_ents[ind].size();
6562
6563
6564 entprocs.initialize( 1, 0, 1, 0, npairs );
6565 entprocs.enableWriteAccess();
6566
6567
6568 for( ind = 0, proc_it = buffProcs.begin(); proc_it != buffProcs.end(); ++proc_it, ind++ )
6569 {
6570 for( Range::iterator rit = sent_ents[ind].begin(); rit != sent_ents[ind].end(); ++rit )
6571 {
6572 entprocs.vi_wr[entprocs.get_n()] = *proc_it;
6573 entprocs.vul_wr[entprocs.get_n()] = *rit;
6574 entprocs.inc_n();
6575 }
6576 }
6577
6578 moab::TupleList::buffer sort_buffer;
6579 sort_buffer.buffer_init( npairs );
6580 entprocs.sort( 1, &sort_buffer );
6581
6582 entprocs.disableWriteAccess();
6583 sort_buffer.reset();
6584
6585 return MB_SUCCESS;
6586 }
6587
6588 ErrorCode ParallelComm::exchange_ghost_cells( ParallelComm** pcs,
6589 unsigned int num_procs,
6590 int ghost_dim,
6591 int bridge_dim,
6592 int num_layers,
6593 int addl_ents,
6594 bool store_remote_handles,
6595 EntityHandle* file_sets )
6596 {
6597
6598
6599
6600
6601
6602 assert( num_layers > 0 || store_remote_handles );
6603
6604 const bool is_iface = !num_layers;
6605
6606 unsigned int ind;
6607 ParallelComm* pc;
6608 ErrorCode result = MB_SUCCESS;
6609
6610 std::vector< Error* > ehs( num_procs );
6611 for( unsigned int i = 0; i < num_procs; i++ )
6612 {
6613 result = pcs[i]->get_moab()->query_interface( ehs[i] );
6614 assert( MB_SUCCESS == result );
6615 }
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626 Range sent_ents[MAX_SHARING_PROCS][MAX_SHARING_PROCS], allsent[MAX_SHARING_PROCS];
6627
6628
6629
6630
6631 TupleList entprocs[MAX_SHARING_PROCS];
6632 for( unsigned int p = 0; p < num_procs; p++ )
6633 {
6634 pc = pcs[p];
6635 result = pc->get_sent_ents( is_iface, bridge_dim, ghost_dim, num_layers, addl_ents, sent_ents[p], allsent[p],
6636 entprocs[p] );MB_CHK_SET_ERR( result, "p = " << p << ", get_sent_ents failed" );
6637
6638
6639
6640
6641 for( ind = 0; ind < pc->buffProcs.size(); ind++ )
6642 {
6643
6644 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
6645 result = pc->pack_entities( sent_ents[p][ind], pc->localOwnedBuffs[ind], store_remote_handles,
6646 pc->buffProcs[ind], is_iface, &entprocs[p], &allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", packing entities failed" );
6647 }
6648
6649 entprocs[p].reset();
6650 }
6651
6652
6653
6654
6655
6656
6657 std::vector< std::vector< EntityHandle > > L1hloc[MAX_SHARING_PROCS], L1hrem[MAX_SHARING_PROCS];
6658 std::vector< std::vector< int > > L1p[MAX_SHARING_PROCS];
6659 std::vector< EntityHandle > L2hloc[MAX_SHARING_PROCS], L2hrem[MAX_SHARING_PROCS];
6660 std::vector< unsigned int > L2p[MAX_SHARING_PROCS];
6661 std::vector< EntityHandle > new_ents[MAX_SHARING_PROCS];
6662
6663 for( unsigned int p = 0; p < num_procs; p++ )
6664 {
6665 L1hloc[p].resize( pcs[p]->buffProcs.size() );
6666 L1hrem[p].resize( pcs[p]->buffProcs.size() );
6667 L1p[p].resize( pcs[p]->buffProcs.size() );
6668 }
6669
6670 for( unsigned int p = 0; p < num_procs; p++ )
6671 {
6672 pc = pcs[p];
6673
6674 for( ind = 0; ind < pc->buffProcs.size(); ind++ )
6675 {
6676
6677
6678
6679
6680
6681
6682 if( pc->localOwnedBuffs[ind]->get_stored_size() == 0 ) continue;
6683
6684 unsigned int to_p = pc->buffProcs[ind];
6685 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
6686 result = pcs[to_p]->unpack_entities( pc->localOwnedBuffs[ind]->buff_ptr, store_remote_handles, ind,
6687 is_iface, L1hloc[to_p], L1hrem[to_p], L1p[to_p], L2hloc[to_p],
6688 L2hrem[to_p], L2p[to_p], new_ents[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack entities" );
6689 }
6690 }
6691
6692 if( is_iface )
6693 {
6694
6695
6696
6697 for( unsigned int p = 0; p < num_procs; p++ )
6698 {
6699 result = pcs[p]->check_clean_iface( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
6700 }
6701
6702 #ifndef NDEBUG
6703 for( unsigned int p = 0; p < num_procs; p++ )
6704 {
6705 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
6706 }
6707 result = check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
6708 #endif
6709 return MB_SUCCESS;
6710 }
6711
6712
6713
6714
6715
6716 std::vector< unsigned int >::iterator proc_it;
6717 for( unsigned int p = 0; p < num_procs; p++ )
6718 {
6719 pc = pcs[p];
6720
6721 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
6722 {
6723
6724 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
6725 result = pc->pack_remote_handles( L1hloc[p][ind], L1hrem[p][ind], L1p[p][ind], *proc_it,
6726 pc->localOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to pack remote handles" );
6727 }
6728 }
6729
6730
6731
6732
6733 for( unsigned int p = 0; p < num_procs; p++ )
6734 {
6735 pc = pcs[p];
6736
6737 for( ind = 0, proc_it = pc->buffProcs.begin(); proc_it != pc->buffProcs.end(); ++proc_it, ind++ )
6738 {
6739
6740 unsigned int to_p = pc->buffProcs[ind];
6741 pc->localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
6742 result = pcs[to_p]->unpack_remote_handles( p, pc->localOwnedBuffs[ind]->buff_ptr, L2hloc[to_p],
6743 L2hrem[to_p], L2p[to_p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to unpack remote handles" );
6744 }
6745 }
6746
6747 #ifndef NDEBUG
6748 for( unsigned int p = 0; p < num_procs; p++ )
6749 {
6750 result = pcs[p]->check_sent_ents( allsent[p] );MB_CHK_SET_ERR( result, "p = " << p << ", failed to check on shared entities" );
6751 }
6752
6753 result = ParallelComm::check_all_shared_handles( pcs, num_procs );MB_CHK_SET_ERR( result, "Failed to check on all shared handles" );
6754 #endif
6755
6756 if( file_sets )
6757 {
6758 for( unsigned int p = 0; p < num_procs; p++ )
6759 {
6760 if( new_ents[p].empty() ) continue;
6761 result = pcs[p]->get_moab()->add_entities( file_sets[p], &new_ents[p][0], new_ents[p].size() );MB_CHK_SET_ERR( result, "p = " << p << ", failed to add new entities to set" );
6762 }
6763 }
6764
6765 return MB_SUCCESS;
6766 }
6767
6768 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& exchange_procs )
6769 {
6770
6771 int n_proc = exchange_procs.size();
6772 for( int i = 0; i < n_proc; i++ )
6773 get_buffers( exchange_procs[i] );
6774 reset_all_buffers();
6775
6776
6777
6778 int success;
6779 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6780 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6781 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6782
6783 int incoming = 0;
6784 for( int i = 0; i < n_proc; i++ )
6785 {
6786 int ind = get_buffers( exchange_procs[i] );
6787 incoming++;
6788 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
6789 MB_MESG_ENTS_SIZE, incoming );
6790 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
6791 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
6792 if( success != MPI_SUCCESS )
6793 {
6794 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
6795 }
6796 }
6797
6798 return MB_SUCCESS;
6799 }
6800
6801 ErrorCode ParallelComm::post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs )
6802 {
6803
6804 int num = shared_procs.size();
6805 for( int i = 0; i < num; i++ )
6806 get_buffers( shared_procs[i] );
6807 reset_all_buffers();
6808 num = remoteOwnedBuffs.size();
6809 for( int i = 0; i < num; i++ )
6810 remoteOwnedBuffs[i]->set_stored_size();
6811 num = localOwnedBuffs.size();
6812 for( int i = 0; i < num; i++ )
6813 localOwnedBuffs[i]->set_stored_size();
6814
6815
6816
6817 int success;
6818 recvReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6819 recvRemotehReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6820 sendReqs.resize( 2 * buffProcs.size(), MPI_REQUEST_NULL );
6821
6822 int incoming = 0;
6823 std::set< unsigned int >::iterator it = recv_procs.begin();
6824 std::set< unsigned int >::iterator eit = recv_procs.end();
6825 for( ; it != eit; ++it )
6826 {
6827 int ind = get_buffers( *it );
6828 incoming++;
6829 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
6830 MB_MESG_ENTS_SIZE, incoming );
6831 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
6832 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recvReqs[2 * ind] );
6833 if( success != MPI_SUCCESS )
6834 {
6835 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
6836 }
6837 }
6838
6839 return MB_SUCCESS;
6840 }
6841
6842 ErrorCode ParallelComm::exchange_owned_meshs( std::vector< unsigned int >& exchange_procs,
6843 std::vector< Range* >& exchange_ents,
6844 std::vector< MPI_Request >& recv_ent_reqs,
6845 std::vector< MPI_Request >& recv_remoteh_reqs,
6846 bool store_remote_handles,
6847 bool wait_all,
6848 bool migrate,
6849 int dim )
6850 {
6851
6852
6853 ErrorCode result;
6854 std::vector< unsigned int > exchange_procs_sets;
6855 std::vector< Range* > exchange_sets;
6856 int n_proc = exchange_procs.size();
6857 for( int i = 0; i < n_proc; i++ )
6858 {
6859 Range set_range = exchange_ents[i]->subset_by_type( MBENTITYSET );
6860 *exchange_ents[i] = subtract( *exchange_ents[i], set_range );
6861 Range* tmp_range = new Range( set_range );
6862 exchange_sets.push_back( tmp_range );
6863 exchange_procs_sets.push_back( exchange_procs[i] );
6864 }
6865
6866 if( dim == 2 )
6867 {
6868
6869 result = exchange_owned_mesh( exchange_procs, exchange_ents, recvReqs, recvRemotehReqs, true,
6870 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" );
6871
6872
6873 result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recvReqs, recvRemotehReqs, false,
6874 store_remote_handles, wait_all, migrate );
6875 }
6876 else
6877 {
6878
6879 result = exchange_owned_mesh( exchange_procs, exchange_ents, recv_ent_reqs, recv_remoteh_reqs, false,
6880 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh entities" );
6881
6882
6883 result = exchange_owned_mesh( exchange_procs_sets, exchange_sets, recv_ent_reqs, recv_remoteh_reqs, false,
6884 store_remote_handles, wait_all, migrate );MB_CHK_SET_ERR( result, "Failed to exchange owned mesh sets" );
6885 }
6886
6887 for( int i = 0; i < n_proc; i++ )
6888 delete exchange_sets[i];
6889
6890
6891 std::map< std::vector< int >, std::vector< EntityHandle > > proc_nvecs;
6892 int procs[MAX_SHARING_PROCS];
6893 EntityHandle handles[MAX_SHARING_PROCS];
6894 int nprocs;
6895 unsigned char pstat;
6896 for( std::set< EntityHandle >::iterator vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
6897 {
6898 if( mbImpl->dimension_from_handle( *vit ) > 2 ) continue;
6899 result = get_sharing_data( *vit, procs, handles, pstat, nprocs );MB_CHK_SET_ERR( result, "Failed to get sharing data in exchange_owned_meshs" );
6900 std::sort( procs, procs + nprocs );
6901 std::vector< int > tmp_procs( procs, procs + nprocs );
6902 assert( tmp_procs.size() != 2 );
6903 proc_nvecs[tmp_procs].push_back( *vit );
6904 }
6905
6906
6907 result = create_interface_sets( proc_nvecs );MB_CHK_SET_ERR( result, "Failed to create interface sets" );
6908
6909 return MB_SUCCESS;
6910 }
6911
6912 ErrorCode ParallelComm::exchange_owned_mesh( std::vector< unsigned int >& exchange_procs,
6913 std::vector< Range* >& exchange_ents,
6914 std::vector< MPI_Request >& recv_ent_reqs,
6915 std::vector< MPI_Request >& recv_remoteh_reqs,
6916 const bool recv_posted,
6917 bool store_remote_handles,
6918 bool wait_all,
6919 bool migrate )
6920 {
6921 #ifdef MOAB_HAVE_MPE
6922 if( myDebug->get_verbosity() == 2 )
6923 {
6924 MPE_Log_event( OWNED_START, procConfig.proc_rank(), "Starting owned ents exchange." );
6925 }
6926 #endif
6927
6928 myDebug->tprintf( 1, "Entering exchange_owned_mesh\n" );
6929 if( myDebug->get_verbosity() == 4 )
6930 {
6931 msgs.clear();
6932 msgs.reserve( MAX_SHARING_PROCS );
6933 }
6934 unsigned int i;
6935 int ind, success;
6936 ErrorCode result = MB_SUCCESS;
6937 int incoming1 = 0, incoming2 = 0;
6938
6939
6940 unsigned int n_proc = exchange_procs.size();
6941 for( i = 0; i < n_proc; i++ )
6942 {
6943 ind = get_buffers( exchange_procs[i] );
6944 result = add_verts( *exchange_ents[i] );MB_CHK_SET_ERR( result, "Failed to add verts" );
6945
6946
6947 Range tmp_range;
6948 result = filter_pstatus( *exchange_ents[i], PSTATUS_SHARED, PSTATUS_AND, buffProcs[ind], &tmp_range );MB_CHK_SET_ERR( result, "Failed to filter on owner" );
6949 if( !tmp_range.empty() )
6950 {
6951 *exchange_ents[i] = subtract( *exchange_ents[i], tmp_range );
6952 }
6953 }
6954
6955
6956
6957
6958 #ifdef MOAB_HAVE_MPE
6959 if( myDebug->get_verbosity() == 2 )
6960 {
6961 MPE_Log_event( ENTITIES_START, procConfig.proc_rank(), "Starting entity exchange." );
6962 }
6963 #endif
6964
6965
6966 if( !recv_posted )
6967 {
6968 reset_all_buffers();
6969 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
6970 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
6971 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
6972
6973 for( i = 0; i < n_proc; i++ )
6974 {
6975 ind = get_buffers( exchange_procs[i] );
6976 incoming1++;
6977 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[ind], remoteOwnedBuffs[ind]->mem_ptr,
6978 INITIAL_BUFF_SIZE, MB_MESG_ENTS_SIZE, incoming1 );
6979 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, buffProcs[ind],
6980 MB_MESG_ENTS_SIZE, procConfig.proc_comm(), &recv_ent_reqs[3 * ind] );
6981 if( success != MPI_SUCCESS )
6982 {
6983 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in owned entity exchange" );
6984 }
6985 }
6986 }
6987 else
6988 incoming1 += n_proc;
6989
6990
6991
6992
6993
6994 Range allsent, tmp_range;
6995 int dum_ack_buff;
6996 int npairs = 0;
6997 TupleList entprocs;
6998 for( i = 0; i < n_proc; i++ )
6999 {
7000 int n_ents = exchange_ents[i]->size();
7001 if( n_ents > 0 )
7002 {
7003 npairs += n_ents;
7004 allsent.merge( *exchange_ents[i] );
7005 }
7006 }
7007
7008
7009 entprocs.initialize( 1, 0, 1, 0, npairs );
7010 entprocs.enableWriteAccess();
7011
7012
7013 for( i = 0; i < n_proc; i++ )
7014 {
7015 for( Range::iterator rit = exchange_ents[i]->begin(); rit != exchange_ents[i]->end(); ++rit )
7016 {
7017 entprocs.vi_wr[entprocs.get_n()] = exchange_procs[i];
7018 entprocs.vul_wr[entprocs.get_n()] = *rit;
7019 entprocs.inc_n();
7020 }
7021 }
7022
7023
7024 moab::TupleList::buffer sort_buffer;
7025 sort_buffer.buffer_init( npairs );
7026 entprocs.sort( 1, &sort_buffer );
7027 sort_buffer.reset();
7028
7029 myDebug->tprintf( 1, "allsent ents compactness (size) = %f (%lu)\n", allsent.compactness(),
7030 (unsigned long)allsent.size() );
7031
7032
7033
7034
7035 for( i = 0; i < n_proc; i++ )
7036 {
7037 ind = get_buffers( exchange_procs[i] );
7038 myDebug->tprintf( 1, "Sent ents compactness (size) = %f (%lu)\n", exchange_ents[i]->compactness(),
7039 (unsigned long)exchange_ents[i]->size() );
7040
7041 localOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
7042 result = pack_buffer( *exchange_ents[i], false, true, store_remote_handles, buffProcs[ind],
7043 localOwnedBuffs[ind], &entprocs, &allsent );
7044
7045 if( myDebug->get_verbosity() == 4 )
7046 {
7047 msgs.resize( msgs.size() + 1 );
7048 msgs.back() = new Buffer( *localOwnedBuffs[ind] );
7049 }
7050
7051
7052 result = send_buffer( exchange_procs[i], localOwnedBuffs[ind], MB_MESG_ENTS_SIZE, sendReqs[3 * ind],
7053 recv_ent_reqs[3 * ind + 2], &dum_ack_buff, incoming1, MB_MESG_REMOTEH_SIZE,
7054 ( store_remote_handles ? localOwnedBuffs[ind] : NULL ), &recv_remoteh_reqs[3 * ind],
7055 &incoming2 );MB_CHK_SET_ERR( result, "Failed to Isend in ghost exchange" );
7056 }
7057
7058 entprocs.reset();
7059
7060
7061
7062
7063
7064 MPI_Status status;
7065 std::vector< std::vector< EntityHandle > > recd_ents( buffProcs.size() );
7066 std::vector< std::vector< EntityHandle > > L1hloc( buffProcs.size() ), L1hrem( buffProcs.size() );
7067 std::vector< std::vector< int > > L1p( buffProcs.size() );
7068 std::vector< EntityHandle > L2hloc, L2hrem;
7069 std::vector< unsigned int > L2p;
7070 std::vector< EntityHandle > new_ents;
7071
7072 while( incoming1 )
7073 {
7074
7075
7076 PRINT_DEBUG_WAITANY( recv_ent_reqs, MB_MESG_ENTS_SIZE, procConfig.proc_rank() );
7077
7078 success = MPI_Waitany( 3 * buffProcs.size(), &recv_ent_reqs[0], &ind, &status );
7079 if( MPI_SUCCESS != success )
7080 {
7081 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
7082 }
7083
7084 PRINT_DEBUG_RECD( status );
7085
7086
7087 incoming1--;
7088 bool done = false;
7089
7090
7091 unsigned int base_ind = 3 * ( ind / 3 );
7092 result = recv_buffer( MB_MESG_ENTS_SIZE, status, remoteOwnedBuffs[ind / 3], recv_ent_reqs[base_ind + 1],
7093 recv_ent_reqs[base_ind + 2], incoming1, localOwnedBuffs[ind / 3], sendReqs[base_ind + 1],
7094 sendReqs[base_ind + 2], done, ( store_remote_handles ? localOwnedBuffs[ind / 3] : NULL ),
7095 MB_MESG_REMOTEH_SIZE, &recv_remoteh_reqs[base_ind + 1], &incoming2 );MB_CHK_SET_ERR( result, "Failed to receive buffer" );
7096
7097 if( done )
7098 {
7099 if( myDebug->get_verbosity() == 4 )
7100 {
7101 msgs.resize( msgs.size() + 1 );
7102 msgs.back() = new Buffer( *remoteOwnedBuffs[ind / 3] );
7103 }
7104
7105
7106 remoteOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
7107 result = unpack_buffer( remoteOwnedBuffs[ind / 3]->buff_ptr, store_remote_handles, buffProcs[ind / 3],
7108 ind / 3, L1hloc, L1hrem, L1p, L2hloc, L2hrem, L2p, new_ents, true );
7109 if( MB_SUCCESS != result )
7110 {
7111 std::cout << "Failed to unpack entities. Buffer contents:" << std::endl;
7112 print_buffer( remoteOwnedBuffs[ind / 3]->mem_ptr, MB_MESG_ENTS_SIZE, buffProcs[ind / 3], false );
7113 return result;
7114 }
7115
7116 if( recv_ent_reqs.size() != 3 * buffProcs.size() )
7117 {
7118
7119 recv_remoteh_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7120 for( i = recv_ent_reqs.size(); i < 3 * buffProcs.size(); i += 3 )
7121 {
7122 localOwnedBuffs[i / 3]->reset_buffer();
7123 incoming2++;
7124 PRINT_DEBUG_IRECV( procConfig.proc_rank(), buffProcs[i / 3], localOwnedBuffs[i / 3]->mem_ptr,
7125 INITIAL_BUFF_SIZE, MB_MESG_REMOTEH_SIZE, incoming2 );
7126 success = MPI_Irecv( localOwnedBuffs[i / 3]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR,
7127 buffProcs[i / 3], MB_MESG_REMOTEH_SIZE, procConfig.proc_comm(),
7128 &recv_remoteh_reqs[i] );
7129 if( success != MPI_SUCCESS )
7130 {
7131 MB_SET_ERR( MB_FAILURE, "Failed to post irecv for remote handles in ghost exchange" );
7132 }
7133 }
7134 recv_ent_reqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7135 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7136 }
7137 }
7138 }
7139
7140
7141 result = assign_entities_part( new_ents, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to assign entities to part" );
7142 if( migrate )
7143 {
7144 result = remove_entities_part( allsent, procConfig.proc_rank() );MB_CHK_SET_ERR( result, "Failed to remove entities to part" );
7145 }
7146
7147
7148 if( recv_ent_reqs.size() != 3 * buffProcs.size() )
7149 {
7150
7151 MB_SET_ERR( MB_FAILURE, "Requests length doesn't match proc count in entity exchange" );
7152 }
7153
7154 #ifdef MOAB_HAVE_MPE
7155 if( myDebug->get_verbosity() == 2 )
7156 {
7157 MPE_Log_event( ENTITIES_END, procConfig.proc_rank(), "Ending entity exchange." );
7158 }
7159 #endif
7160
7161
7162 if( wait_all )
7163 {
7164 if( myDebug->get_verbosity() == 5 )
7165 {
7166 success = MPI_Barrier( procConfig.proc_comm() );
7167 }
7168 else
7169 {
7170 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
7171 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
7172 if( MPI_SUCCESS != success )
7173 {
7174 MB_SET_ERR( MB_FAILURE, "Failed in waitall in exchange owned mesh" );
7175 }
7176 }
7177 }
7178
7179
7180
7181
7182 for( i = 0; i < n_proc; i++ )
7183 {
7184 ind = get_buffers( exchange_procs[i] );
7185
7186 remoteOwnedBuffs[ind]->reset_buffer( sizeof( int ) );
7187
7188 result = pack_remote_handles( L1hloc[ind], L1hrem[ind], L1p[ind], buffProcs[ind], remoteOwnedBuffs[ind] );MB_CHK_SET_ERR( result, "Failed to pack remote handles" );
7189 remoteOwnedBuffs[ind]->set_stored_size();
7190
7191 if( myDebug->get_verbosity() == 4 )
7192 {
7193 msgs.resize( msgs.size() + 1 );
7194 msgs.back() = new Buffer( *remoteOwnedBuffs[ind] );
7195 }
7196 result = send_buffer( buffProcs[ind], remoteOwnedBuffs[ind], MB_MESG_REMOTEH_SIZE, sendReqs[3 * ind],
7197 recv_remoteh_reqs[3 * ind + 2], &dum_ack_buff, incoming2 );MB_CHK_SET_ERR( result, "Failed to send remote handles" );
7198 }
7199
7200
7201
7202
7203 while( incoming2 )
7204 {
7205 PRINT_DEBUG_WAITANY( recv_remoteh_reqs, MB_MESG_REMOTEH_SIZE, procConfig.proc_rank() );
7206 success = MPI_Waitany( 3 * buffProcs.size(), &recv_remoteh_reqs[0], &ind, &status );
7207 if( MPI_SUCCESS != success )
7208 {
7209 MB_SET_ERR( MB_FAILURE, "Failed in waitany in owned entity exchange" );
7210 }
7211
7212
7213 incoming2--;
7214
7215 PRINT_DEBUG_RECD( status );
7216
7217 bool done = false;
7218 unsigned int base_ind = 3 * ( ind / 3 );
7219 result = recv_buffer( MB_MESG_REMOTEH_SIZE, status, localOwnedBuffs[ind / 3], recv_remoteh_reqs[base_ind + 1],
7220 recv_remoteh_reqs[base_ind + 2], incoming2, remoteOwnedBuffs[ind / 3],
7221 sendReqs[base_ind + 1], sendReqs[base_ind + 2], done );MB_CHK_SET_ERR( result, "Failed to receive remote handles" );
7222
7223 if( done )
7224 {
7225
7226 if( myDebug->get_verbosity() == 4 )
7227 {
7228 msgs.resize( msgs.size() + 1 );
7229 msgs.back() = new Buffer( *localOwnedBuffs[ind / 3] );
7230 }
7231
7232 localOwnedBuffs[ind / 3]->reset_ptr( sizeof( int ) );
7233 result =
7234 unpack_remote_handles( buffProcs[ind / 3], localOwnedBuffs[ind / 3]->buff_ptr, L2hloc, L2hrem, L2p );MB_CHK_SET_ERR( result, "Failed to unpack remote handles" );
7235 }
7236 }
7237
7238 #ifdef MOAB_HAVE_MPE
7239 if( myDebug->get_verbosity() == 2 )
7240 {
7241 MPE_Log_event( RHANDLES_END, procConfig.proc_rank(), "Ending remote handles." );
7242 MPE_Log_event( OWNED_END, procConfig.proc_rank(), "Ending ghost exchange (still doing checks)." );
7243 }
7244 #endif
7245
7246
7247
7248
7249 if( wait_all )
7250 {
7251 if( myDebug->get_verbosity() == 5 )
7252 {
7253 success = MPI_Barrier( procConfig.proc_comm() );
7254 }
7255 else
7256 {
7257 MPI_Status mult_status[3 * MAX_SHARING_PROCS];
7258 success = MPI_Waitall( 3 * buffProcs.size(), &recv_remoteh_reqs[0], mult_status );
7259 if( MPI_SUCCESS == success ) success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], mult_status );
7260 }
7261 if( MPI_SUCCESS != success )
7262 {
7263 MB_SET_ERR( MB_FAILURE, "Failed in waitall in owned entity exchange" );
7264 }
7265 }
7266
7267 #ifndef NDEBUG
7268 result = check_sent_ents( allsent );MB_CHK_SET_ERR( result, "Failed check on shared entities" );
7269 #endif
7270 myDebug->tprintf( 1, "Exiting exchange_owned_mesh\n" );
7271
7272 return MB_SUCCESS;
7273 }
7274
7275 ErrorCode ParallelComm::get_iface_entities( int other_proc, int dim, Range& iface_ents )
7276 {
7277 Range iface_sets;
7278 ErrorCode result = MB_SUCCESS;
7279
7280 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
7281 {
7282 if( -1 != other_proc && !is_iface_proc( *rit, other_proc ) ) continue;
7283
7284 if( -1 == dim )
7285 {
7286 result = mbImpl->get_entities_by_handle( *rit, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" );
7287 }
7288 else
7289 {
7290 result = mbImpl->get_entities_by_dimension( *rit, dim, iface_ents );MB_CHK_SET_ERR( result, "Failed to get entities in iface set" );
7291 }
7292 }
7293
7294 return MB_SUCCESS;
7295 }
7296
7297 ErrorCode ParallelComm::assign_entities_part( std::vector< EntityHandle >& entities, const int proc )
7298 {
7299 EntityHandle part_set;
7300 ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" );
7301
7302 if( part_set > 0 )
7303 {
7304 result = mbImpl->add_entities( part_set, &entities[0], entities.size() );MB_CHK_SET_ERR( result, "Failed to add entities to part set" );
7305 }
7306
7307 return MB_SUCCESS;
7308 }
7309
7310 ErrorCode ParallelComm::remove_entities_part( Range& entities, const int proc )
7311 {
7312 EntityHandle part_set;
7313 ErrorCode result = get_part_handle( proc, part_set );MB_CHK_SET_ERR( result, "Failed to get part handle" );
7314
7315 if( part_set > 0 )
7316 {
7317 result = mbImpl->remove_entities( part_set, entities );MB_CHK_SET_ERR( result, "Failed to remove entities from part set" );
7318 }
7319
7320 return MB_SUCCESS;
7321 }
7322
7323 ErrorCode ParallelComm::check_sent_ents( Range& allsent )
7324 {
7325
7326
7327 std::vector< unsigned char > pstat( allsent.size() );
7328 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), allsent, &pstat[0] );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
7329 std::vector< EntityHandle > handles( allsent.size() );
7330 result = mbImpl->tag_get_data( sharedh_tag(), allsent, &handles[0] );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
7331 std::vector< int > procs( allsent.size() );
7332 result = mbImpl->tag_get_data( sharedp_tag(), allsent, &procs[0] );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
7333
7334 Range bad_entities;
7335
7336 Range::iterator rit;
7337 unsigned int i;
7338 EntityHandle dum_hs[MAX_SHARING_PROCS];
7339 int dum_ps[MAX_SHARING_PROCS];
7340
7341 for( rit = allsent.begin(), i = 0; rit != allsent.end(); ++rit, i++ )
7342 {
7343 if( -1 != procs[i] && 0 == handles[i] )
7344 bad_entities.insert( *rit );
7345 else
7346 {
7347
7348 result = mbImpl->tag_get_data( sharedps_tag(), &( *rit ), 1, dum_ps );
7349 if( MB_TAG_NOT_FOUND == result )
7350 continue;
7351 else if( MB_SUCCESS != result )
7352 MB_SET_ERR( result, "Failed to get sharedps tag data" );
7353 result = mbImpl->tag_get_data( sharedhs_tag(), &( *rit ), 1, dum_hs );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
7354
7355
7356 int* ns_proc = std::find( dum_ps, dum_ps + MAX_SHARING_PROCS, -1 );
7357 int num_procs = ns_proc - dum_ps;
7358 assert( num_procs <= MAX_SHARING_PROCS );
7359
7360 EntityHandle* ns_handle = std::find( dum_hs, dum_hs + num_procs, 0 );
7361 int num_handles = ns_handle - dum_hs;
7362 assert( num_handles <= num_procs );
7363 if( num_handles != num_procs ) bad_entities.insert( *rit );
7364 }
7365 }
7366
7367 return MB_SUCCESS;
7368 }
7369
7370 ErrorCode ParallelComm::pack_remote_handles( std::vector< EntityHandle >& L1hloc,
7371 std::vector< EntityHandle >& L1hrem,
7372 std::vector< int >& L1p,
7373 unsigned int ,
7374 Buffer* buff )
7375 {
7376 assert( std::find( L1hloc.begin(), L1hloc.end(), (EntityHandle)0 ) == L1hloc.end() );
7377
7378
7379 buff->check_space( ( ( L1p.size() + 1 ) * sizeof( int ) + ( L1hloc.size() + 1 ) * sizeof( EntityHandle ) +
7380 ( L1hrem.size() + 1 ) * sizeof( EntityHandle ) ) );
7381
7382
7383 PACK_INT( buff->buff_ptr, L1hloc.size() );
7384 PACK_INTS( buff->buff_ptr, &L1p[0], L1p.size() );
7385
7386
7387 PACK_EH( buff->buff_ptr, &L1hrem[0], L1hrem.size() );
7388 PACK_EH( buff->buff_ptr, &L1hloc[0], L1hloc.size() );
7389
7390 buff->set_stored_size();
7391
7392 return MB_SUCCESS;
7393 }
7394
7395 ErrorCode ParallelComm::unpack_remote_handles( unsigned int from_proc,
7396 unsigned char*& buff_ptr,
7397 std::vector< EntityHandle >& L2hloc,
7398 std::vector< EntityHandle >& L2hrem,
7399 std::vector< unsigned int >& L2p )
7400 {
7401
7402 int num_eh;
7403 UNPACK_INT( buff_ptr, num_eh );
7404
7405 unsigned char* buff_proc = buff_ptr;
7406 buff_ptr += num_eh * sizeof( int );
7407 unsigned char* buff_rem = buff_ptr + num_eh * sizeof( EntityHandle );
7408 ErrorCode result;
7409 EntityHandle hpair[2], new_h;
7410 int proc;
7411 for( int i = 0; i < num_eh; i++ )
7412 {
7413 UNPACK_INT( buff_proc, proc );
7414
7415
7416
7417 UNPACK_EH( buff_ptr, hpair, 1 );
7418 UNPACK_EH( buff_rem, hpair + 1, 1 );
7419
7420 if( -1 != proc )
7421 {
7422 result = find_existing_entity( false, proc, hpair[0], 3, NULL, 0, mbImpl->type_from_handle( hpair[1] ),
7423 L2hloc, L2hrem, L2p, new_h );MB_CHK_SET_ERR( result, "Didn't get existing entity" );
7424 if( new_h )
7425 hpair[0] = new_h;
7426 else
7427 hpair[0] = 0;
7428 }
7429 if( !( hpair[0] && hpair[1] ) ) return MB_FAILURE;
7430 int this_proc = from_proc;
7431 result = update_remote_data( hpair[0], &this_proc, hpair + 1, 1, 0 );MB_CHK_SET_ERR( result, "Failed to set remote data range on sent entities in ghost exchange" );
7432 }
7433
7434 return MB_SUCCESS;
7435 }
7436
7437 ErrorCode ParallelComm::get_ghosted_entities( int bridge_dim,
7438 int ghost_dim,
7439 int to_proc,
7440 int num_layers,
7441 int addl_ents,
7442 Range& ghosted_ents )
7443 {
7444
7445 Range from_ents;
7446 ErrorCode result = MB_SUCCESS;
7447 assert( 0 < num_layers );
7448 for( Range::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); ++rit )
7449 {
7450 if( !is_iface_proc( *rit, to_proc ) ) continue;
7451
7452
7453 if( bridge_dim == -1 )
7454 {
7455 result = mbImpl->get_entities_by_handle( *rit, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" );
7456 }
7457 else
7458 {
7459 result = mbImpl->get_entities_by_dimension( *rit, bridge_dim, from_ents );MB_CHK_SET_ERR( result, "Failed to get bridge ents in the set" );
7460 }
7461
7462
7463 if( from_ents.empty() ) continue;
7464 result =
7465 MeshTopoUtil( mbImpl ).get_bridge_adjacencies( from_ents, bridge_dim, ghost_dim, ghosted_ents, num_layers );MB_CHK_SET_ERR( result, "Failed to get bridge adjacencies" );
7466 }
7467
7468 result = add_verts( ghosted_ents );MB_CHK_SET_ERR( result, "Failed to add verts" );
7469
7470 if( addl_ents )
7471 {
7472
7473 Range tmp_ents, tmp_owned, tmp_notowned;
7474 tmp_owned = ghosted_ents.subset_by_dimension( ghost_dim );
7475 if( tmp_owned.empty() ) return result;
7476
7477 tmp_notowned = tmp_owned;
7478
7479
7480 result = filter_pstatus( tmp_owned, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &tmp_owned );MB_CHK_SET_ERR( result, "Failed to filter owned entities" );
7481
7482 tmp_notowned -= tmp_owned;
7483
7484
7485 if( 1 == addl_ents || 3 == addl_ents )
7486 {
7487 result = mbImpl->get_adjacencies( tmp_owned, 1, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for owned ghost entities" );
7488 result = mbImpl->get_adjacencies( tmp_notowned, 1, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get edge adjacencies for notowned ghost entities" );
7489 }
7490 if( 2 == addl_ents || 3 == addl_ents )
7491 {
7492 result = mbImpl->get_adjacencies( tmp_owned, 2, true, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for owned ghost entities" );
7493 result = mbImpl->get_adjacencies( tmp_notowned, 2, false, tmp_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get face adjacencies for notowned ghost entities" );
7494 }
7495
7496 ghosted_ents.merge( tmp_ents );
7497 }
7498
7499 return result;
7500 }
7501
7502 ErrorCode ParallelComm::add_verts( Range& sent_ents )
7503 {
7504
7505
7506
7507 std::pair< Range::const_iterator, Range::const_iterator > set_range = sent_ents.equal_range( MBENTITYSET );
7508 ErrorCode result = MB_SUCCESS, tmp_result;
7509 for( Range::const_iterator rit = set_range.first; rit != set_range.second; ++rit )
7510 {
7511 tmp_result = mbImpl->get_entities_by_type( *rit, MBVERTEX, sent_ents );MB_CHK_SET_ERR( tmp_result, "Failed to get contained verts" );
7512 }
7513
7514
7515 Range tmp_ents;
7516 std::copy( sent_ents.begin(), set_range.first, range_inserter( tmp_ents ) );
7517 result = mbImpl->get_adjacencies( tmp_ents, 0, false, sent_ents, Interface::UNION );MB_CHK_SET_ERR( result, "Failed to get vertices adj to ghosted ents" );
7518
7519
7520 Range polyhedra = sent_ents.subset_by_type( MBPOLYHEDRON );
7521
7522 result = mbImpl->get_connectivity( polyhedra, sent_ents );MB_CHK_SET_ERR( result, "Failed to get polyhedra faces" );
7523 return result;
7524 }
7525
7526 ErrorCode ParallelComm::exchange_tags( const std::vector< Tag >& src_tags,
7527 const std::vector< Tag >& dst_tags,
7528 const Range& entities_in )
7529 {
7530 ErrorCode result;
7531 int success;
7532
7533 myDebug->tprintf( 1, "Entering exchange_tags\n" );
7534
7535
7536 std::set< unsigned int > exch_procs;
7537 result = get_comm_procs( exch_procs );
7538
7539
7540
7541 std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7542
7543 std::vector< unsigned int >::iterator sit;
7544 int ind;
7545
7546 reset_all_buffers();
7547 int incoming = 0;
7548
7549 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
7550 {
7551 incoming++;
7552 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
7553 MB_MESG_TAGS_SIZE, incoming );
7554
7555 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
7556 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] );
7557 if( success != MPI_SUCCESS )
7558 {
7559 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
7560 }
7561 }
7562
7563
7564
7565 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7566
7567
7568 Range entities;
7569 if( entities_in.empty() )
7570 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
7571 else
7572 entities = entities_in;
7573
7574 int dum_ack_buff;
7575
7576 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
7577 {
7578 Range tag_ents = entities;
7579
7580
7581 result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
7582
7583
7584 if( !tag_ents.empty() )
7585 {
7586 result = filter_pstatus( tag_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed pstatus NOT check" );
7587 }
7588
7589
7590 std::vector< Range > tag_ranges;
7591 for( std::vector< Tag >::const_iterator vit = src_tags.begin(); vit != src_tags.end(); ++vit )
7592 {
7593 const void* ptr;
7594 int sz;
7595 if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS )
7596 {
7597 Range tagged_ents;
7598 mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents );
7599 tag_ranges.push_back( intersect( tag_ents, tagged_ents ) );
7600 }
7601 else
7602 {
7603 tag_ranges.push_back( tag_ents );
7604 }
7605 }
7606
7607
7608
7609 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
7610
7611 result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" );
7612
7613
7614 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
7615 recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
7616 }
7617
7618
7619 while( incoming )
7620 {
7621 MPI_Status status;
7622 int index_in_recv_requests;
7623 PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
7624 success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status );
7625 if( MPI_SUCCESS != success )
7626 {
7627 MB_SET_ERR( MB_FAILURE, "Failed in waitany in tag exchange" );
7628 }
7629
7630 ind = index_in_recv_requests / 3;
7631
7632 PRINT_DEBUG_RECD( status );
7633
7634
7635 incoming--;
7636
7637 bool done = false;
7638 std::vector< EntityHandle > dum_vec;
7639 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
7640 recv_tag_reqs[3 * ind + 1],
7641 recv_tag_reqs[3 * ind + 2],
7642
7643 incoming, localOwnedBuffs[ind],
7644 sendReqs[3 * ind + 1],
7645 sendReqs[3 * ind + 2],
7646 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
7647 if( done )
7648 {
7649 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
7650 result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind] );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" );
7651 }
7652 }
7653
7654
7655 if( myDebug->get_verbosity() == 5 )
7656 {
7657 success = MPI_Barrier( procConfig.proc_comm() );
7658 }
7659 else
7660 {
7661 MPI_Status status[3 * MAX_SHARING_PROCS];
7662 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
7663 }
7664 if( MPI_SUCCESS != success )
7665 {
7666 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
7667 }
7668
7669
7670
7671 assert( src_tags.size() == dst_tags.size() );
7672 if( src_tags != dst_tags )
7673 {
7674 std::vector< unsigned char > data;
7675 Range owned_ents;
7676 if( entities_in.empty() )
7677 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
7678 else
7679 owned_ents = entities_in;
7680 result = filter_pstatus( owned_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failure to get subset of owned entities" );
7681
7682 if( !owned_ents.empty() )
7683 {
7684
7685 for( size_t i = 0; i < src_tags.size(); i++ )
7686 {
7687 if( src_tags[i] == dst_tags[i] ) continue;
7688
7689 Range tagged_ents( owned_ents );
7690 result = mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &src_tags[0], 0, 1, tagged_ents,
7691 Interface::INTERSECT );MB_CHK_SET_ERR( result, "get_entities_by_type_and_tag(type == MBMAXTYPE) failed" );
7692
7693 int sz, size2;
7694 result = mbImpl->tag_get_bytes( src_tags[i], sz );MB_CHK_SET_ERR( result, "tag_get_size failed" );
7695 result = mbImpl->tag_get_bytes( dst_tags[i], size2 );MB_CHK_SET_ERR( result, "tag_get_size failed" );
7696 if( sz != size2 )
7697 {
7698 MB_SET_ERR( MB_FAILURE, "tag sizes don't match" );
7699 }
7700
7701 data.resize( sz * tagged_ents.size() );
7702 result = mbImpl->tag_get_data( src_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_get_data failed" );
7703 result = mbImpl->tag_set_data( dst_tags[i], tagged_ents, &data[0] );MB_CHK_SET_ERR( result, "tag_set_data failed" );
7704 }
7705 }
7706 }
7707
7708 myDebug->tprintf( 1, "Exiting exchange_tags" );
7709
7710 return MB_SUCCESS;
7711 }
7712
7713 ErrorCode ParallelComm::reduce_tags( const std::vector< Tag >& src_tags,
7714 const std::vector< Tag >& dst_tags,
7715 const MPI_Op mpi_op,
7716 const Range& entities_in )
7717 {
7718 ErrorCode result;
7719 int success;
7720
7721 myDebug->tprintf( 1, "Entering reduce_tags\n" );
7722
7723
7724 if( src_tags.size() != dst_tags.size() )
7725 {
7726 MB_SET_ERR( MB_FAILURE, "Source and destination tag handles must be specified for reduce_tags" );
7727 }
7728
7729
7730 std::vector< Tag >::const_iterator vits, vitd;
7731 int tags_size, tagd_size;
7732 DataType tags_type, tagd_type;
7733 std::vector< unsigned char > vals;
7734 std::vector< int > tags_sizes;
7735 for( vits = src_tags.begin(), vitd = dst_tags.begin(); vits != src_tags.end(); ++vits, ++vitd )
7736 {
7737
7738 result = mbImpl->tag_get_data_type( *vits, tags_type );MB_CHK_SET_ERR( result, "Failed to get src tag data type" );
7739 if( tags_type != MB_TYPE_INTEGER && tags_type != MB_TYPE_DOUBLE && tags_type != MB_TYPE_BIT )
7740 {
7741 MB_SET_ERR( MB_FAILURE, "Src/dst tags must have integer, double, or bit data type" );
7742 }
7743
7744 result = mbImpl->tag_get_bytes( *vits, tags_size );MB_CHK_SET_ERR( result, "Failed to get src tag bytes" );
7745 vals.resize( tags_size );
7746 result = mbImpl->tag_get_default_value( *vits, &vals[0] );MB_CHK_SET_ERR( result, "Src tag must have default value" );
7747
7748 tags_sizes.push_back( tags_size );
7749
7750
7751 if( *vits == *vitd ) continue;
7752
7753 result = mbImpl->tag_get_bytes( *vitd, tagd_size );MB_CHK_SET_ERR( result, "Coudln't get dst tag bytes" );
7754 if( tags_size != tagd_size )
7755 {
7756 MB_SET_ERR( MB_FAILURE, "Sizes between src and dst tags don't match" );
7757 }
7758 result = mbImpl->tag_get_data_type( *vitd, tagd_type );MB_CHK_SET_ERR( result, "Coudln't get dst tag data type" );
7759 if( tags_type != tagd_type )
7760 {
7761 MB_SET_ERR( MB_FAILURE, "Src and dst tags must be of same data type" );
7762 }
7763 }
7764
7765
7766 std::set< unsigned int > exch_procs;
7767 result = get_comm_procs( exch_procs );
7768
7769
7770
7771 std::vector< MPI_Request > recv_tag_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7772
7773 std::vector< unsigned int >::iterator sit;
7774 int ind;
7775
7776 reset_all_buffers();
7777 int incoming = 0;
7778
7779 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
7780 {
7781 incoming++;
7782 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
7783 MB_MESG_TAGS_SIZE, incoming );
7784
7785 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
7786 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_tag_reqs[3 * ind] );
7787 if( success != MPI_SUCCESS )
7788 {
7789 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in ghost exchange" );
7790 }
7791 }
7792
7793
7794
7795 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
7796
7797
7798 Range entities;
7799 if( entities_in.empty() )
7800 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( entities ) );
7801 else
7802 entities = entities_in;
7803
7804
7805 std::vector< Tag >::const_iterator vit = src_tags.begin(), vit2 = dst_tags.begin();
7806 std::vector< int >::const_iterator vsizes = tags_sizes.begin();
7807 for( ; vit != src_tags.end(); ++vit, ++vit2, ++vsizes )
7808 {
7809 if( *vit == *vit2 ) continue;
7810 vals.resize( entities.size() * ( *vsizes ) );
7811 result = mbImpl->tag_get_data( *vit, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't get data properly" );
7812 result = mbImpl->tag_set_data( *vit2, entities, &vals[0] );MB_CHK_SET_ERR( result, "Didn't set data properly" );
7813 }
7814
7815 int dum_ack_buff;
7816
7817 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
7818 {
7819 Range tag_ents = entities;
7820
7821
7822 result = filter_pstatus( tag_ents, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
7823
7824
7825 std::vector< Range > tag_ranges;
7826 for( vit = src_tags.begin(); vit != src_tags.end(); ++vit )
7827 {
7828 const void* ptr;
7829 int sz;
7830 if( mbImpl->tag_get_default_value( *vit, ptr, sz ) != MB_SUCCESS )
7831 {
7832 Range tagged_ents;
7833 mbImpl->get_entities_by_type_and_tag( 0, MBMAXTYPE, &*vit, 0, 1, tagged_ents );
7834 tag_ranges.push_back( intersect( tag_ents, tagged_ents ) );
7835 }
7836 else
7837 tag_ranges.push_back( tag_ents );
7838 }
7839
7840
7841
7842 localOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
7843
7844 result = pack_tags( tag_ents, src_tags, dst_tags, tag_ranges, localOwnedBuffs[ind], true, *sit );MB_CHK_SET_ERR( result, "Failed to count buffer in pack_send_tag" );
7845
7846
7847 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
7848 recv_tag_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
7849 }
7850
7851
7852 while( incoming )
7853 {
7854 MPI_Status status;
7855 int index_in_recv_requests;
7856 PRINT_DEBUG_WAITANY( recv_tag_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
7857 success = MPI_Waitany( 3 * buffProcs.size(), &recv_tag_reqs[0], &index_in_recv_requests, &status );
7858 if( MPI_SUCCESS != success )
7859 {
7860 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
7861 }
7862 ind = index_in_recv_requests / 3;
7863
7864 PRINT_DEBUG_RECD( status );
7865
7866
7867 incoming--;
7868
7869 bool done = false;
7870 std::vector< EntityHandle > dum_vec;
7871 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
7872 recv_tag_reqs[3 * ind + 1],
7873 recv_tag_reqs[3 * ind + 2],
7874
7875 incoming, localOwnedBuffs[ind],
7876 sendReqs[3 * ind + 1],
7877 sendReqs[3 * ind + 2],
7878 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
7879 if( done )
7880 {
7881 remoteOwnedBuffs[ind]->reset_ptr( sizeof( int ) );
7882 result = unpack_tags( remoteOwnedBuffs[ind]->buff_ptr, dum_vec, true, buffProcs[ind], &mpi_op );MB_CHK_SET_ERR( result, "Failed to recv-unpack-tag message" );
7883 }
7884 }
7885
7886
7887 if( myDebug->get_verbosity() == 5 )
7888 {
7889 success = MPI_Barrier( procConfig.proc_comm() );
7890 }
7891 else
7892 {
7893 MPI_Status status[3 * MAX_SHARING_PROCS];
7894 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
7895 }
7896 if( MPI_SUCCESS != success )
7897 {
7898 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
7899 }
7900
7901 myDebug->tprintf( 1, "Exiting reduce_tags" );
7902
7903 return MB_SUCCESS;
7904 }
7905
7906
7907 Tag ParallelComm::sharedp_tag()
7908 {
7909 if( !sharedpTag )
7910 {
7911 int def_val = -1;
7912 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROC_TAG_NAME, 1, MB_TYPE_INTEGER, sharedpTag,
7913 MB_TAG_DENSE | MB_TAG_CREAT, &def_val );
7914 if( MB_SUCCESS != result ) return 0;
7915 }
7916
7917 return sharedpTag;
7918 }
7919
7920
7921 Tag ParallelComm::sharedps_tag()
7922 {
7923 if( !sharedpsTag )
7924 {
7925 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_PROCS_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_INTEGER,
7926 sharedpsTag, MB_TAG_SPARSE | MB_TAG_CREAT );
7927 if( MB_SUCCESS != result ) return 0;
7928 }
7929
7930 return sharedpsTag;
7931 }
7932
7933
7934 Tag ParallelComm::sharedh_tag()
7935 {
7936 if( !sharedhTag )
7937 {
7938 EntityHandle def_val = 0;
7939 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLE_TAG_NAME, 1, MB_TYPE_HANDLE, sharedhTag,
7940 MB_TAG_DENSE | MB_TAG_CREAT, &def_val );
7941 if( MB_SUCCESS != result ) return 0;
7942 }
7943
7944 return sharedhTag;
7945 }
7946
7947
7948 Tag ParallelComm::sharedhs_tag()
7949 {
7950 if( !sharedhsTag )
7951 {
7952 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_SHARED_HANDLES_TAG_NAME, MAX_SHARING_PROCS, MB_TYPE_HANDLE,
7953 sharedhsTag, MB_TAG_SPARSE | MB_TAG_CREAT );
7954 if( MB_SUCCESS != result ) return 0;
7955 }
7956
7957 return sharedhsTag;
7958 }
7959
7960
7961 Tag ParallelComm::pstatus_tag()
7962 {
7963 if( !pstatusTag )
7964 {
7965 unsigned char tmp_pstatus = 0;
7966 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_STATUS_TAG_NAME, 1, MB_TYPE_OPAQUE, pstatusTag,
7967 MB_TAG_DENSE | MB_TAG_CREAT, &tmp_pstatus );
7968 if( MB_SUCCESS != result ) return 0;
7969 }
7970
7971 return pstatusTag;
7972 }
7973
7974
7975 Tag ParallelComm::partition_tag()
7976 {
7977 if( !partitionTag )
7978 {
7979 int dum_id = -1;
7980 ErrorCode result = mbImpl->tag_get_handle( PARALLEL_PARTITION_TAG_NAME, 1, MB_TYPE_INTEGER, partitionTag,
7981 MB_TAG_SPARSE | MB_TAG_CREAT, &dum_id );
7982 if( MB_SUCCESS != result ) return 0;
7983 }
7984
7985 return partitionTag;
7986 }
7987
7988
7989 Tag ParallelComm::pcomm_tag( Interface* impl, bool create_if_missing )
7990 {
7991 Tag this_tag = 0;
7992 ErrorCode result;
7993 if( create_if_missing )
7994 {
7995 result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ),
7996 MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE | MB_TAG_CREAT );
7997 }
7998 else
7999 {
8000 result = impl->tag_get_handle( PARALLEL_COMM_TAG_NAME, MAX_SHARING_PROCS * sizeof( ParallelComm* ),
8001 MB_TYPE_OPAQUE, this_tag, MB_TAG_SPARSE );
8002 }
8003
8004 if( MB_SUCCESS != result ) return 0;
8005
8006 return this_tag;
8007 }
8008
8009
8010 ParallelComm* ParallelComm::get_pcomm( Interface* impl, const int index )
8011 {
8012 Tag pc_tag = pcomm_tag( impl, false );
8013 if( 0 == pc_tag ) return NULL;
8014
8015 const EntityHandle root = 0;
8016 ParallelComm* pc_array[MAX_SHARING_PROCS];
8017 ErrorCode result = impl->tag_get_data( pc_tag, &root, 1, (void*)pc_array );
8018 if( MB_SUCCESS != result ) return NULL;
8019
8020 return pc_array[index];
8021 }
8022
8023 ErrorCode ParallelComm::get_all_pcomm( Interface* impl, std::vector< ParallelComm* >& list )
8024 {
8025 Tag pc_tag = pcomm_tag( impl, false );
8026 if( 0 == pc_tag ) return MB_TAG_NOT_FOUND;
8027
8028 const EntityHandle root = 0;
8029 ParallelComm* pc_array[MAX_SHARING_PROCS];
8030 ErrorCode rval = impl->tag_get_data( pc_tag, &root, 1, pc_array );
8031 if( MB_SUCCESS != rval ) return rval;
8032
8033 for( int i = 0; i < MAX_SHARING_PROCS; i++ )
8034 {
8035 if( pc_array[i] ) list.push_back( pc_array[i] );
8036 }
8037
8038 return MB_SUCCESS;
8039 }
8040
8041
8042 ParallelComm* ParallelComm::get_pcomm( Interface* impl, EntityHandle prtn, const MPI_Comm* comm )
8043 {
8044 ErrorCode rval;
8045 ParallelComm* result = 0;
8046
8047 Tag prtn_tag;
8048 rval =
8049 impl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag, MB_TAG_SPARSE | MB_TAG_CREAT );
8050 if( MB_SUCCESS != rval ) return 0;
8051
8052 int pcomm_id;
8053 rval = impl->tag_get_data( prtn_tag, &prtn, 1, &pcomm_id );
8054 if( MB_SUCCESS == rval )
8055 {
8056 result = get_pcomm( impl, pcomm_id );
8057 }
8058 else if( MB_TAG_NOT_FOUND == rval && comm )
8059 {
8060 result = new ParallelComm( impl, *comm, &pcomm_id );
8061 if( !result ) return 0;
8062 result->set_partitioning( prtn );
8063
8064 rval = impl->tag_set_data( prtn_tag, &prtn, 1, &pcomm_id );
8065 if( MB_SUCCESS != rval )
8066 {
8067 delete result;
8068 result = 0;
8069 }
8070 }
8071
8072 return result;
8073 }
8074
8075 ErrorCode ParallelComm::set_partitioning( EntityHandle set )
8076 {
8077 ErrorCode rval;
8078 Tag prtn_tag;
8079 rval = mbImpl->tag_get_handle( PARTITIONING_PCOMM_TAG_NAME, 1, MB_TYPE_INTEGER, prtn_tag,
8080 MB_TAG_SPARSE | MB_TAG_CREAT );
8081 if( MB_SUCCESS != rval ) return rval;
8082
8083
8084 ParallelComm* pcomm_arr[MAX_SHARING_PROCS];
8085 Tag pc_tag = pcomm_tag( mbImpl, false );
8086 if( 0 == pc_tag ) return MB_FAILURE;
8087 const EntityHandle root = 0;
8088 ErrorCode result = mbImpl->tag_get_data( pc_tag, &root, 1, pcomm_arr );
8089 if( MB_SUCCESS != result ) return MB_FAILURE;
8090 int id = std::find( pcomm_arr, pcomm_arr + MAX_SHARING_PROCS, this ) - pcomm_arr;
8091 if( id == MAX_SHARING_PROCS ) return MB_FAILURE;
8092
8093 EntityHandle old = partitioningSet;
8094 if( old )
8095 {
8096 rval = mbImpl->tag_delete_data( prtn_tag, &old, 1 );
8097 if( MB_SUCCESS != rval ) return rval;
8098 partitioningSet = 0;
8099 }
8100
8101 if( !set ) return MB_SUCCESS;
8102
8103 Range contents;
8104 if( old )
8105 {
8106 rval = mbImpl->get_entities_by_handle( old, contents );
8107 if( MB_SUCCESS != rval ) return rval;
8108 }
8109 else
8110 {
8111 contents = partition_sets();
8112 }
8113
8114 rval = mbImpl->add_entities( set, contents );
8115 if( MB_SUCCESS != rval ) return rval;
8116
8117
8118 rval = mbImpl->tag_set_data( prtn_tag, &set, 1, &id );
8119 if( MB_SUCCESS != rval ) return rval;
8120
8121 partitioningSet = set;
8122 return MB_SUCCESS;
8123 }
8124
8125
8126 ErrorCode ParallelComm::get_part_entities( Range& ents, int dim )
8127 {
8128 ErrorCode result;
8129
8130 for( Range::iterator rit = partitionSets.begin(); rit != partitionSets.end(); ++rit )
8131 {
8132 Range tmp_ents;
8133 if( -1 == dim )
8134 result = mbImpl->get_entities_by_handle( *rit, tmp_ents, true );
8135 else
8136 result = mbImpl->get_entities_by_dimension( *rit, dim, tmp_ents, true );
8137
8138 if( MB_SUCCESS != result ) return result;
8139 ents.merge( tmp_ents );
8140 }
8141
8142 return MB_SUCCESS;
8143 }
8144
8145
8147 ErrorCode ParallelComm::get_owner_handle( EntityHandle entity, int& owner, EntityHandle& handle )
8148 {
8149 unsigned char pstat;
8150 int sharing_procs[MAX_SHARING_PROCS];
8151 EntityHandle sharing_handles[MAX_SHARING_PROCS];
8152
8153 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
8154 if( !( pstat & PSTATUS_NOT_OWNED ) )
8155 {
8156 owner = proc_config().proc_rank();
8157 handle = entity;
8158 }
8159 else if( pstat & PSTATUS_MULTISHARED )
8160 {
8161 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedps tag data" );
8162 owner = sharing_procs[0];
8163 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedhs tag data" );
8164 handle = sharing_handles[0];
8165 }
8166 else if( pstat & PSTATUS_SHARED )
8167 {
8168 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, sharing_procs );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
8169 owner = sharing_procs[0];
8170 result = mbImpl->tag_get_data( sharedh_tag(), &entity, 1, sharing_handles );MB_CHK_SET_ERR( result, "Failed to get sharedh tag data" );
8171 handle = sharing_handles[0];
8172 }
8173 else
8174 {
8175 owner = -1;
8176 handle = 0;
8177 }
8178
8179 return MB_SUCCESS;
8180 }
8181
8182 ErrorCode ParallelComm::get_global_part_count( int& count_out ) const
8183 {
8184 count_out = globalPartCount;
8185 return count_out < 0 ? MB_FAILURE : MB_SUCCESS;
8186 }
8187
8188 ErrorCode ParallelComm::get_part_owner( int part_id, int& owner ) const
8189 {
8190
8191 owner = part_id;
8192 return MB_SUCCESS;
8193 }
8194
8195 ErrorCode ParallelComm::get_part_id( EntityHandle , int& id_out ) const
8196 {
8197
8198 id_out = proc_config().proc_rank();
8199 return MB_SUCCESS;
8200 }
8201
8202 ErrorCode ParallelComm::get_part_handle( int id, EntityHandle& handle_out ) const
8203 {
8204
8205 if( (unsigned)id != proc_config().proc_rank() ) return MB_ENTITY_NOT_FOUND;
8206 handle_out = partition_sets().front();
8207 return MB_SUCCESS;
8208 }
8209
8210 ErrorCode ParallelComm::create_part( EntityHandle& set_out )
8211 {
8212
8213 globalPartCount = -1;
8214
8215
8216 ErrorCode rval = mbImpl->create_meshset( MESHSET_SET, set_out );
8217 if( MB_SUCCESS != rval ) return rval;
8218
8219
8220 int val = proc_config().proc_rank();
8221 rval = mbImpl->tag_set_data( part_tag(), &set_out, 1, &val );
8222
8223 if( MB_SUCCESS != rval )
8224 {
8225 mbImpl->delete_entities( &set_out, 1 );
8226 return rval;
8227 }
8228
8229 if( get_partitioning() )
8230 {
8231 rval = mbImpl->add_entities( get_partitioning(), &set_out, 1 );
8232 if( MB_SUCCESS != rval )
8233 {
8234 mbImpl->delete_entities( &set_out, 1 );
8235 return rval;
8236 }
8237 }
8238
8239 moab::Range& pSets = this->partition_sets();
8240 if( pSets.index( set_out ) < 0 )
8241 {
8242 pSets.insert( set_out );
8243 }
8244
8245 return MB_SUCCESS;
8246 }
8247
8248 ErrorCode ParallelComm::destroy_part( EntityHandle part_id )
8249 {
8250
8251 globalPartCount = -1;
8252
8253 ErrorCode rval;
8254 if( get_partitioning() )
8255 {
8256 rval = mbImpl->remove_entities( get_partitioning(), &part_id, 1 );
8257 if( MB_SUCCESS != rval ) return rval;
8258 }
8259
8260 moab::Range& pSets = this->partition_sets();
8261 if( pSets.index( part_id ) >= 0 )
8262 {
8263 pSets.erase( part_id );
8264 }
8265 return mbImpl->delete_entities( &part_id, 1 );
8266 }
8267
8268 ErrorCode ParallelComm::collective_sync_partition()
8269 {
8270 int count = partition_sets().size();
8271 globalPartCount = 0;
8272 int err = MPI_Allreduce( &count, &globalPartCount, 1, MPI_INT, MPI_SUM, proc_config().proc_comm() );
8273 return err ? MB_FAILURE : MB_SUCCESS;
8274 }
8275
8276 ErrorCode ParallelComm::get_part_neighbor_ids( EntityHandle part,
8277 int neighbors_out[MAX_SHARING_PROCS],
8278 int& num_neighbors_out )
8279 {
8280 ErrorCode rval;
8281 Range iface;
8282 rval = get_interface_sets( part, iface );
8283 if( MB_SUCCESS != rval ) return rval;
8284
8285 num_neighbors_out = 0;
8286 int n, j = 0;
8287 int tmp[MAX_SHARING_PROCS] = { 0 }, curr[MAX_SHARING_PROCS] = { 0 };
8288 int* parts[2] = { neighbors_out, tmp };
8289 for( Range::iterator i = iface.begin(); i != iface.end(); ++i )
8290 {
8291 unsigned char pstat;
8292 rval = get_sharing_data( *i, curr, NULL, pstat, n );
8293 if( MB_SUCCESS != rval ) return rval;
8294 std::sort( curr, curr + n );
8295 assert( num_neighbors_out < MAX_SHARING_PROCS );
8296 int* k = std::set_union( parts[j], parts[j] + num_neighbors_out, curr, curr + n, parts[1 - j] );
8297 j = 1 - j;
8298 num_neighbors_out = k - parts[j];
8299 }
8300 if( parts[j] != neighbors_out ) std::copy( parts[j], parts[j] + num_neighbors_out, neighbors_out );
8301
8302
8303 int id;
8304 rval = get_part_id( part, id );
8305 if( MB_SUCCESS == rval )
8306 num_neighbors_out = std::remove( neighbors_out, neighbors_out + num_neighbors_out, id ) - neighbors_out;
8307 return rval;
8308 }
8309
8310 ErrorCode ParallelComm::get_interface_sets( EntityHandle, Range& iface_sets_out, int* adj_part_id )
8311 {
8312
8313
8314
8315 iface_sets_out = interface_sets();
8316
8317 if( adj_part_id )
8318 {
8319 int part_ids[MAX_SHARING_PROCS], num_parts;
8320 Range::iterator i = iface_sets_out.begin();
8321 while( i != iface_sets_out.end() )
8322 {
8323 unsigned char pstat;
8324 ErrorCode rval = get_sharing_data( *i, part_ids, NULL, pstat, num_parts );
8325 if( MB_SUCCESS != rval ) return rval;
8326
8327 if( std::find( part_ids, part_ids + num_parts, *adj_part_id ) - part_ids != num_parts )
8328 ++i;
8329 else
8330 i = iface_sets_out.erase( i );
8331 }
8332 }
8333
8334 return MB_SUCCESS;
8335 }
8336
8337 ErrorCode ParallelComm::get_owning_part( EntityHandle handle, int& owning_part_id, EntityHandle* remote_handle )
8338 {
8339
8340
8341
8342 unsigned char pstat;
8343 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &handle, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
8344 if( !( pstat & PSTATUS_NOT_OWNED ) )
8345 {
8346 owning_part_id = proc_config().proc_rank();
8347 if( remote_handle ) *remote_handle = handle;
8348 return MB_SUCCESS;
8349 }
8350
8351
8352
8353 result = mbImpl->tag_get_data( sharedp_tag(), &handle, 1, &owning_part_id );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
8354 if( owning_part_id != -1 )
8355 {
8356
8357 if( !remote_handle ) return MB_SUCCESS;
8358
8359
8360 return mbImpl->tag_get_data( sharedh_tag(), &handle, 1, remote_handle );
8361 }
8362
8363
8364
8365 const void* part_id_list = 0;
8366 result = mbImpl->tag_get_by_ptr( sharedps_tag(), &handle, 1, &part_id_list );
8367 if( MB_SUCCESS != result ) return result;
8368 owning_part_id = ( (const int*)part_id_list )[0];
8369
8370
8371 if( !remote_handle ) return MB_SUCCESS;
8372
8373
8374 const void* handle_list = 0;
8375 result = mbImpl->tag_get_by_ptr( sharedhs_tag(), &handle, 1, &handle_list );
8376 if( MB_SUCCESS != result ) return result;
8377
8378 *remote_handle = ( (const EntityHandle*)handle_list )[0];
8379 return MB_SUCCESS;
8380 }
8381
8382 ErrorCode ParallelComm::get_sharing_parts( EntityHandle entity,
8383 int part_ids_out[MAX_SHARING_PROCS],
8384 int& num_part_ids_out,
8385 EntityHandle remote_handles[MAX_SHARING_PROCS] )
8386 {
8387
8388
8389
8390 unsigned char pstat;
8391 ErrorCode result = mbImpl->tag_get_data( pstatus_tag(), &entity, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to get pstatus tag data" );
8392 if( !( pstat & PSTATUS_SHARED ) )
8393 {
8394 part_ids_out[0] = proc_config().proc_rank();
8395 if( remote_handles ) remote_handles[0] = entity;
8396 num_part_ids_out = 1;
8397 return MB_SUCCESS;
8398 }
8399
8400
8401
8402 result = mbImpl->tag_get_data( sharedp_tag(), &entity, 1, part_ids_out );MB_CHK_SET_ERR( result, "Failed to get sharedp tag data" );
8403 if( part_ids_out[0] != -1 )
8404 {
8405 num_part_ids_out = 2;
8406 part_ids_out[1] = proc_config().proc_rank();
8407
8408
8409 if( !remote_handles ) return MB_SUCCESS;
8410
8411
8412 remote_handles[1] = entity;
8413 return mbImpl->tag_get_data( sharedh_tag(), &entity, 1, remote_handles );
8414 }
8415
8416
8417
8418 result = mbImpl->tag_get_data( sharedps_tag(), &entity, 1, part_ids_out );
8419 if( MB_SUCCESS != result ) return result;
8420
8421 for( num_part_ids_out = 0; num_part_ids_out < MAX_SHARING_PROCS && part_ids_out[num_part_ids_out] >= 0;
8422 num_part_ids_out++ )
8423 ;
8424
8425 #ifndef NDEBUG
8426 int my_idx = std::find( part_ids_out, part_ids_out + num_part_ids_out, proc_config().proc_rank() ) - part_ids_out;
8427 assert( my_idx < num_part_ids_out );
8428 #endif
8429
8430
8431 if( !remote_handles ) return MB_SUCCESS;
8432
8433
8434 result = mbImpl->tag_get_data( sharedhs_tag(), &entity, 1, remote_handles );
8435
8436 assert( remote_handles[my_idx] == entity );
8437
8438 return result;
8439 }
8440
8441 ErrorCode ParallelComm::pack_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data )
8442 {
8443
8444 ErrorCode rval = MB_SUCCESS;
8445 int ent_procs[MAX_SHARING_PROCS];
8446 EntityHandle handles[MAX_SHARING_PROCS];
8447 int num_sharing, tmp_int;
8448 SharedEntityData tmp;
8449 send_data.resize( buffProcs.size() );
8450 for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i )
8451 {
8452 tmp.remote = *i;
8453 rval = get_owner( *i, tmp_int );
8454 tmp.owner = tmp_int;
8455 if( MB_SUCCESS != rval ) return rval;
8456
8457 unsigned char pstat;
8458 rval = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing );
8459 if( MB_SUCCESS != rval ) return rval;
8460 for( int j = 0; j < num_sharing; j++ )
8461 {
8462 if( ent_procs[j] == (int)proc_config().proc_rank() ) continue;
8463 tmp.local = handles[j];
8464 int ind = get_buffers( ent_procs[j] );
8465 assert( -1 != ind );
8466 if( (int)send_data.size() < ind + 1 ) send_data.resize( ind + 1 );
8467 send_data[ind].push_back( tmp );
8468 }
8469 }
8470
8471 return MB_SUCCESS;
8472 }
8473
8474 ErrorCode ParallelComm::exchange_all_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data,
8475 std::vector< std::vector< SharedEntityData > >& result )
8476 {
8477 int ierr;
8478 const int tag = 0;
8479 const MPI_Comm cm = procConfig.proc_comm();
8480 const int num_proc = buffProcs.size();
8481 const std::vector< int > procs( buffProcs.begin(), buffProcs.end() );
8482 std::vector< MPI_Request > recv_req( buffProcs.size(), MPI_REQUEST_NULL );
8483 std::vector< MPI_Request > send_req( buffProcs.size(), MPI_REQUEST_NULL );
8484
8485
8486 std::vector< int > sizes_send( num_proc ), sizes_recv( num_proc );
8487 for( int i = 0; i < num_proc; i++ )
8488 {
8489 ierr = MPI_Irecv( &sizes_recv[i], 1, MPI_INT, procs[i], tag, cm, &recv_req[i] );
8490 if( ierr ) return MB_FILE_WRITE_ERROR;
8491 }
8492
8493
8494 assert( num_proc == (int)send_data.size() );
8495
8496 result.resize( num_proc );
8497 for( int i = 0; i < num_proc; i++ )
8498 {
8499 sizes_send[i] = send_data[i].size();
8500 ierr = MPI_Isend( &sizes_send[i], 1, MPI_INT, buffProcs[i], tag, cm, &send_req[i] );
8501 if( ierr ) return MB_FILE_WRITE_ERROR;
8502 }
8503
8504
8505 std::vector< MPI_Status > stat( num_proc );
8506 ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
8507 if( ierr ) return MB_FILE_WRITE_ERROR;
8508
8509
8510 ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
8511 if( ierr ) return MB_FILE_WRITE_ERROR;
8512
8513
8514 for( int i = 0; i < num_proc; i++ )
8515 {
8516 result[i].resize( sizes_recv[i] );
8517 ierr = MPI_Irecv( (void*)( &( result[i][0] ) ), sizeof( SharedEntityData ) * sizes_recv[i], MPI_UNSIGNED_CHAR,
8518 buffProcs[i], tag, cm, &recv_req[i] );
8519 if( ierr ) return MB_FILE_WRITE_ERROR;
8520 }
8521
8522
8523 for( int i = 0; i < num_proc; i++ )
8524 {
8525 ierr = MPI_Isend( (void*)( &( send_data[i][0] ) ), sizeof( SharedEntityData ) * sizes_send[i],
8526 MPI_UNSIGNED_CHAR, buffProcs[i], tag, cm, &send_req[i] );
8527 if( ierr ) return MB_FILE_WRITE_ERROR;
8528 }
8529
8530
8531 ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
8532 if( ierr ) return MB_FILE_WRITE_ERROR;
8533
8534
8535 ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
8536 if( ierr ) return MB_FILE_WRITE_ERROR;
8537
8538 return MB_SUCCESS;
8539 }
8540
8541 ErrorCode ParallelComm::check_all_shared_handles( bool print_em )
8542 {
8543
8544 std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() );
8545
8546 ErrorCode result;
8547 bool done = false;
8548
8549 while( !done )
8550 {
8551 result = check_local_shared();
8552 if( MB_SUCCESS != result )
8553 {
8554 done = true;
8555 continue;
8556 }
8557
8558 result = pack_shared_handles( send_data );
8559 if( MB_SUCCESS != result )
8560 {
8561 done = true;
8562 continue;
8563 }
8564
8565 result = exchange_all_shared_handles( send_data, shents );
8566 if( MB_SUCCESS != result )
8567 {
8568 done = true;
8569 continue;
8570 }
8571
8572 if( !shents.empty() ) result = check_my_shared_handles( shents );
8573 done = true;
8574 }
8575
8576 if( MB_SUCCESS != result && print_em )
8577 {
8578 #ifdef MOAB_HAVE_HDF5
8579 std::ostringstream ent_str;
8580 ent_str << "mesh." << procConfig.proc_rank() << ".h5m";
8581 mbImpl->write_mesh( ent_str.str().c_str() );
8582 #endif
8583 }
8584
8585 return result;
8586 }
8587
8588 ErrorCode ParallelComm::check_local_shared()
8589 {
8590
8591
8592
8593
8594
8595
8596
8597 std::vector< EntityHandle > dum_connect;
8598 const EntityHandle* connect;
8599 int num_connect;
8600 int tmp_procs[MAX_SHARING_PROCS];
8601 EntityHandle tmp_hs[MAX_SHARING_PROCS];
8602 std::set< int > tmp_set, vset;
8603 int num_ps;
8604 ErrorCode result;
8605 unsigned char pstat;
8606 std::vector< EntityHandle > bad_ents;
8607 std::vector< std::string > errors;
8608
8609 std::set< EntityHandle >::iterator vit;
8610 for( vit = sharedEnts.begin(); vit != sharedEnts.end(); ++vit )
8611 {
8612
8613 result = get_sharing_data( *vit, tmp_procs, tmp_hs, pstat, num_ps );
8614 if( MB_SUCCESS != result )
8615 {
8616 bad_ents.push_back( *vit );
8617 errors.push_back( std::string( "Failure getting sharing data." ) );
8618 continue;
8619 }
8620
8621 bool bad = false;
8622
8623 if( !( pstat & PSTATUS_SHARED ) )
8624 errors.push_back( std::string( "Entity should be shared but isn't." ) ), bad = true;
8625
8626
8627 if( pstat & PSTATUS_NOT_OWNED && tmp_procs[0] == (int)procConfig.proc_rank() )
8628 errors.push_back( std::string( "Entity not owned but is first proc." ) ), bad = true;
8629
8630
8631 if( !( pstat & PSTATUS_NOT_OWNED ) && pstat & PSTATUS_MULTISHARED &&
8632 ( tmp_procs[0] != (int)procConfig.proc_rank() || tmp_hs[0] != *vit ) )
8633 errors.push_back( std::string( "Entity owned and multishared but not first proc or not first handle." ) ),
8634 bad = true;
8635
8636 if( bad )
8637 {
8638 bad_ents.push_back( *vit );
8639 continue;
8640 }
8641
8642 EntityType type = mbImpl->type_from_handle( *vit );
8643 if( type == MBVERTEX || type == MBENTITYSET ) continue;
8644
8645
8646 int orig_ps = num_ps;
8647 vset.clear();
8648 std::copy( tmp_procs, tmp_procs + num_ps, std::inserter( vset, vset.begin() ) );
8649
8650
8651 result = mbImpl->get_connectivity( *vit, connect, num_connect, false, &dum_connect );
8652 if( MB_SUCCESS != result )
8653 {
8654 bad_ents.push_back( *vit );
8655 errors.push_back( std::string( "Failed to get connectivity." ) );
8656 continue;
8657 }
8658
8659 for( int i = 0; i < num_connect; i++ )
8660 {
8661 result = get_sharing_data( connect[i], tmp_procs, NULL, pstat, num_ps );
8662 if( MB_SUCCESS != result )
8663 {
8664 bad_ents.push_back( *vit );
8665 continue;
8666 }
8667 if( !num_ps )
8668 {
8669 vset.clear();
8670 break;
8671 }
8672 std::sort( tmp_procs, tmp_procs + num_ps );
8673 tmp_set.clear();
8674 std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(),
8675 std::inserter( tmp_set, tmp_set.end() ) );
8676 vset.swap( tmp_set );
8677 if( vset.empty() ) break;
8678 }
8679
8680
8681 tmp_set.clear();
8682 std::set_intersection( tmp_procs, tmp_procs + num_ps, vset.begin(), vset.end(),
8683 std::inserter( tmp_set, tmp_set.end() ) );
8684 if( orig_ps != (int)tmp_set.size() )
8685 {
8686 errors.push_back( std::string( "Vertex proc set not same size as entity proc set." ) );
8687 bad_ents.push_back( *vit );
8688 for( int i = 0; i < num_connect; i++ )
8689 {
8690 bad_ents.push_back( connect[i] );
8691 errors.push_back( std::string( "vertex in connect" ) );
8692 }
8693 }
8694 }
8695
8696 if( !bad_ents.empty() )
8697 {
8698 std::cout << "Found bad entities in check_local_shared, proc rank " << procConfig.proc_rank() << ","
8699 << std::endl;
8700 std::vector< std::string >::iterator sit;
8701 std::vector< EntityHandle >::iterator rit;
8702 for( rit = bad_ents.begin(), sit = errors.begin(); rit != bad_ents.end(); ++rit, ++sit )
8703 {
8704 list_entities( &( *rit ), 1 );
8705 std::cout << "Reason: " << *sit << std::endl;
8706 }
8707 return MB_FAILURE;
8708 }
8709
8710
8711
8712 return MB_SUCCESS;
8713 }
8714
8715 ErrorCode ParallelComm::check_all_shared_handles( ParallelComm** pcs, int num_pcs )
8716 {
8717 std::vector< std::vector< std::vector< SharedEntityData > > > shents, send_data;
8718 ErrorCode result = MB_SUCCESS, tmp_result;
8719
8720
8721 send_data.resize( num_pcs );
8722 for( int p = 0; p < num_pcs; p++ )
8723 {
8724 tmp_result = pcs[p]->pack_shared_handles( send_data[p] );
8725 if( MB_SUCCESS != tmp_result ) result = tmp_result;
8726 }
8727 if( MB_SUCCESS != result ) return result;
8728
8729
8730 shents.resize( num_pcs );
8731 for( int p = 0; p < num_pcs; p++ )
8732 shents[p].resize( pcs[p]->buffProcs.size() );
8733
8734 for( int p = 0; p < num_pcs; p++ )
8735 {
8736 for( unsigned int idx_p = 0; idx_p < pcs[p]->buffProcs.size(); idx_p++ )
8737 {
8738
8739 int to_p = pcs[p]->buffProcs[idx_p];
8740 int top_idx_p = pcs[to_p]->get_buffers( p );
8741 assert( -1 != top_idx_p );
8742 shents[to_p][top_idx_p] = send_data[p][idx_p];
8743 }
8744 }
8745
8746 for( int p = 0; p < num_pcs; p++ )
8747 {
8748 std::ostringstream ostr;
8749 ostr << "Processor " << p << " bad entities:";
8750 tmp_result = pcs[p]->check_my_shared_handles( shents[p], ostr.str().c_str() );
8751 if( MB_SUCCESS != tmp_result ) result = tmp_result;
8752 }
8753
8754 return result;
8755 }
8756
8757 ErrorCode ParallelComm::check_my_shared_handles( std::vector< std::vector< SharedEntityData > >& shents,
8758 const char* prefix )
8759 {
8760
8761
8762 ErrorCode result;
8763 Range all_shared;
8764 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( all_shared ) );
8765 std::vector< EntityHandle > dum_vec;
8766 all_shared.erase( all_shared.upper_bound( MBPOLYHEDRON ), all_shared.end() );
8767
8768 Range bad_ents, local_shared;
8769 std::vector< SharedEntityData >::iterator vit;
8770 unsigned char tmp_pstat;
8771 for( unsigned int i = 0; i < shents.size(); i++ )
8772 {
8773 int other_proc = buffProcs[i];
8774 result = get_shared_entities( other_proc, local_shared );
8775 if( MB_SUCCESS != result ) return result;
8776 for( vit = shents[i].begin(); vit != shents[i].end(); ++vit )
8777 {
8778 EntityHandle localh = vit->local, remoteh = vit->remote, dumh;
8779 local_shared.erase( localh );
8780 result = get_remote_handles( true, &localh, &dumh, 1, other_proc, dum_vec );
8781 if( MB_SUCCESS != result || dumh != remoteh ) bad_ents.insert( localh );
8782 result = get_pstatus( localh, tmp_pstat );
8783 if( MB_SUCCESS != result || ( !( tmp_pstat & PSTATUS_NOT_OWNED ) && (unsigned)vit->owner != rank() ) ||
8784 ( tmp_pstat & PSTATUS_NOT_OWNED && (unsigned)vit->owner == rank() ) )
8785 bad_ents.insert( localh );
8786 }
8787
8788 if( !local_shared.empty() ) bad_ents.merge( local_shared );
8789 }
8790
8791 if( !bad_ents.empty() )
8792 {
8793 if( prefix ) std::cout << prefix << std::endl;
8794 list_entities( bad_ents );
8795 return MB_FAILURE;
8796 }
8797 else
8798 return MB_SUCCESS;
8799 }
8800
8801 ErrorCode ParallelComm::get_shared_entities( int other_proc,
8802 Range& shared_ents,
8803 int dim,
8804 const bool iface,
8805 const bool owned_filter )
8806 {
8807 shared_ents.clear();
8808 ErrorCode result = MB_SUCCESS;
8809
8810
8811 if( -1 != dim )
8812 {
8813 DimensionPair dp = CN::TypeDimensionMap[dim];
8814 Range dum_range;
8815 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( dum_range ) );
8816 shared_ents.merge( dum_range.lower_bound( dp.first ), dum_range.upper_bound( dp.second ) );
8817 }
8818 else
8819 std::copy( sharedEnts.begin(), sharedEnts.end(), range_inserter( shared_ents ) );
8820
8821
8822 if( iface )
8823 {
8824 result = filter_pstatus( shared_ents, PSTATUS_INTERFACE, PSTATUS_AND );MB_CHK_SET_ERR( result, "Failed to filter by iface" );
8825 }
8826
8827
8828 if( owned_filter )
8829 {
8830 result = filter_pstatus( shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT );MB_CHK_SET_ERR( result, "Failed to filter by owned" );
8831 }
8832
8833
8834 if( -1 != other_proc )
8835 {
8836 result = filter_pstatus( shared_ents, PSTATUS_SHARED, PSTATUS_AND, other_proc );MB_CHK_SET_ERR( result, "Failed to filter by proc" );
8837 }
8838
8839 return result;
8840 }
8841
8842 ErrorCode ParallelComm::clean_shared_tags( std::vector< Range* >& exchange_ents )
8843 {
8844 for( unsigned int i = 0; i < exchange_ents.size(); i++ )
8845 {
8846 Range* ents = exchange_ents[i];
8847 int num_ents = ents->size();
8848 Range::iterator it = ents->begin();
8849
8850 for( int n = 0; n < num_ents; n++ )
8851 {
8852 int sharing_proc;
8853 ErrorCode result = mbImpl->tag_get_data( sharedp_tag(), &( *ents->begin() ), 1, &sharing_proc );
8854 if( result != MB_TAG_NOT_FOUND && sharing_proc == -1 )
8855 {
8856 result = mbImpl->tag_delete_data( sharedp_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedp tag data" );
8857 result = mbImpl->tag_delete_data( sharedh_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete sharedh tag data" );
8858 result = mbImpl->tag_delete_data( pstatus_tag(), &( *it ), 1 );MB_CHK_SET_ERR( result, "Failed to delete pstatus tag data" );
8859 }
8860 ++it;
8861 }
8862 }
8863
8864 return MB_SUCCESS;
8865 }
8866
8867 void ParallelComm::set_debug_verbosity( int verb )
8868 {
8869 myDebug->set_verbosity( verb );
8870 }
8871
8872 int ParallelComm::get_debug_verbosity()
8873 {
8874 return myDebug->get_verbosity();
8875 }
8876
8877 ErrorCode ParallelComm::get_entityset_procs( EntityHandle set, std::vector< unsigned >& ranks ) const
8878 {
8879 return sharedSetData->get_sharing_procs( set, ranks );
8880 }
8881
8882 ErrorCode ParallelComm::get_entityset_owner( EntityHandle entity_set,
8883 unsigned& owner_rank,
8884 EntityHandle* remote_handle ) const
8885 {
8886 if( remote_handle )
8887 return sharedSetData->get_owner( entity_set, owner_rank, *remote_handle );
8888 else
8889 return sharedSetData->get_owner( entity_set, owner_rank );
8890 }
8891
8892 ErrorCode ParallelComm::get_entityset_local_handle( unsigned owning_rank,
8893 EntityHandle remote_handle,
8894 EntityHandle& local_handle ) const
8895 {
8896 return sharedSetData->get_local_handle( owning_rank, remote_handle, local_handle );
8897 }
8898
8899 ErrorCode ParallelComm::get_shared_sets( Range& result ) const
8900 {
8901 return sharedSetData->get_shared_sets( result );
8902 }
8903
8904 ErrorCode ParallelComm::get_entityset_owners( std::vector< unsigned >& ranks ) const
8905 {
8906 return sharedSetData->get_owning_procs( ranks );
8907 }
8908
8909 ErrorCode ParallelComm::get_owned_sets( unsigned owning_rank, Range& sets_out ) const
8910 {
8911 return sharedSetData->get_shared_sets( owning_rank, sets_out );
8912 }
8913
8914 ErrorCode ParallelComm::gather_data( Range& gather_ents,
8915 Tag& tag_handle,
8916 Tag id_tag,
8917 EntityHandle gather_set,
8918 int root_proc_rank )
8919 {
8920 int dim = mbImpl->dimension_from_handle( *gather_ents.begin() );
8921 int bytes_per_tag = 0;
8922 ErrorCode rval = mbImpl->tag_get_bytes( tag_handle, bytes_per_tag );
8923 if( rval != MB_SUCCESS ) return rval;
8924
8925 int sz_buffer = sizeof( int ) + gather_ents.size() * ( sizeof( int ) + bytes_per_tag );
8926 void* senddata = malloc( sz_buffer );
8927 ( (int*)senddata )[0] = (int)gather_ents.size();
8928 int* ptr_int = (int*)senddata + 1;
8929 rval = mbImpl->tag_get_data( id_tag, gather_ents, (void*)ptr_int );
8930 if( rval != MB_SUCCESS ) return rval;
8931 ptr_int = (int*)( senddata ) + 1 + gather_ents.size();
8932 rval = mbImpl->tag_get_data( tag_handle, gather_ents, (void*)ptr_int );
8933 if( rval != MB_SUCCESS ) return rval;
8934 std::vector< int > displs( proc_config().proc_size(), 0 );
8935 MPI_Gather( &sz_buffer, 1, MPI_INT, &displs[0], 1, MPI_INT, root_proc_rank, comm() );
8936 std::vector< int > recvcnts( proc_config().proc_size(), 0 );
8937 std::copy( displs.begin(), displs.end(), recvcnts.begin() );
8938 std::partial_sum( displs.begin(), displs.end(), displs.begin() );
8939 std::vector< int >::iterator lastM1 = displs.end() - 1;
8940 std::copy_backward( displs.begin(), lastM1, displs.end() );
8941
8942 displs[0] = 0;
8943
8944 if( (int)rank() != root_proc_rank )
8945 MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, NULL, NULL, NULL, MPI_BYTE, root_proc_rank, comm() );
8946 else
8947 {
8948 Range gents;
8949 mbImpl->get_entities_by_dimension( gather_set, dim, gents );
8950 int recvbuffsz = gents.size() * ( bytes_per_tag + sizeof( int ) ) + proc_config().proc_size() * sizeof( int );
8951 void* recvbuf = malloc( recvbuffsz );
8952 MPI_Gatherv( senddata, sz_buffer, MPI_BYTE, recvbuf, &recvcnts[0], &displs[0], MPI_BYTE, root_proc_rank,
8953 comm() );
8954
8955 void* gvals = NULL;
8956
8957
8958 bool multiple_sequences = false;
8959 if( gents.psize() > 1 )
8960 multiple_sequences = true;
8961 else
8962 {
8963 int count;
8964 rval = mbImpl->tag_iterate( tag_handle, gents.begin(), gents.end(), count, gvals );
8965 assert( NULL != gvals );
8966 assert( count > 0 );
8967 if( (size_t)count != gents.size() )
8968 {
8969 multiple_sequences = true;
8970 gvals = NULL;
8971 }
8972 }
8973
8974
8975 if( multiple_sequences )
8976 {
8977 gvals = malloc( gents.size() * bytes_per_tag );
8978 assert( NULL != gvals );
8979 }
8980
8981 for( int i = 0; i != (int)size(); i++ )
8982 {
8983 int numents = *(int*)( ( (char*)recvbuf ) + displs[i] );
8984 int* id_ptr = (int*)( ( (char*)recvbuf ) + displs[i] + sizeof( int ) );
8985 char* val_ptr = (char*)( id_ptr + numents );
8986 for( int j = 0; j != numents; j++ )
8987 {
8988 int idx = id_ptr[j];
8989 memcpy( (char*)gvals + ( idx - 1 ) * bytes_per_tag, val_ptr + j * bytes_per_tag, bytes_per_tag );
8990 }
8991 }
8992
8993
8994 free( recvbuf );
8995
8996
8997
8998 if( multiple_sequences )
8999 {
9000 Range::iterator iter = gents.begin();
9001 size_t start_idx = 0;
9002 while( iter != gents.end() )
9003 {
9004 int count;
9005 void* ptr;
9006 rval = mbImpl->tag_iterate( tag_handle, iter, gents.end(), count, ptr );
9007 assert( NULL != ptr );
9008 assert( count > 0 );
9009 memcpy( (char*)ptr, (char*)gvals + start_idx * bytes_per_tag, bytes_per_tag * count );
9010
9011 iter += count;
9012 start_idx += count;
9013 }
9014 assert( start_idx == gents.size() );
9015
9016
9017 free( gvals );
9018 }
9019 }
9020
9021
9022 free( senddata );
9023
9024 return MB_SUCCESS;
9025 }
9026
9027
9036 ErrorCode ParallelComm::settle_intersection_points( Range& edges,
9037 Range& shared_edges_owned,
9038 std::vector< std::vector< EntityHandle >* >& extraNodesVec,
9039 double tolerance )
9040 {
9041
9042
9043 ErrorCode result;
9044 int success;
9045
9046 myDebug->tprintf( 1, "Entering settle_intersection_points\n" );
9047
9048
9049 std::set< unsigned int > exch_procs;
9050 result = get_comm_procs( exch_procs );
9051
9052
9053
9054 std::vector< MPI_Request > recv_intx_reqs( 3 * buffProcs.size(), MPI_REQUEST_NULL );
9055 std::vector< unsigned int >::iterator sit;
9056 int ind;
9057
9058 reset_all_buffers();
9059 int incoming = 0;
9060
9061 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
9062 {
9063 incoming++;
9064 PRINT_DEBUG_IRECV( *sit, procConfig.proc_rank(), remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE,
9065 MB_MESG_TAGS_SIZE, incoming );
9066
9067 success = MPI_Irecv( remoteOwnedBuffs[ind]->mem_ptr, INITIAL_BUFF_SIZE, MPI_UNSIGNED_CHAR, *sit,
9068 MB_MESG_TAGS_SIZE, procConfig.proc_comm(), &recv_intx_reqs[3 * ind] );
9069 if( success != MPI_SUCCESS )
9070 {
9071 MB_SET_ERR( MB_FAILURE, "Failed to post irecv in settle intersection point" );
9072 }
9073 }
9074
9075
9076
9077 sendReqs.resize( 3 * buffProcs.size(), MPI_REQUEST_NULL );
9078
9079
9080 Range& entities = shared_edges_owned;
9081
9082 int dum_ack_buff;
9083
9084 for( ind = 0, sit = buffProcs.begin(); sit != buffProcs.end(); ++sit, ind++ )
9085 {
9086 Range edges_to_send = entities;
9087
9088
9089 result = filter_pstatus( edges_to_send, PSTATUS_SHARED, PSTATUS_AND, *sit );MB_CHK_SET_ERR( result, "Failed pstatus AND check" );
9090
9091
9092
9093
9094
9095 Buffer* buff = localOwnedBuffs[ind];
9096 buff->reset_ptr( sizeof( int ) );
9097
9098
9100
9101
9102
9103 std::vector< EntityHandle > dum_remote_edges( edges_to_send.size() );
9104
9111
9112
9113 std::vector< EntityHandle > dum_vec;
9114 result = get_remote_handles( true, edges_to_send, &dum_remote_edges[0], *sit, dum_vec );MB_CHK_SET_ERR( result, "Failed to get remote handles" );
9115 int count = 4;
9116 count += sizeof( int ) * (int)edges_to_send.size();
9117 count += sizeof( EntityHandle ) * (int)edges_to_send.size();
9118 for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit )
9119 {
9120 EntityHandle edge = *eit;
9121 unsigned int indx = edges.find( edge ) - edges.begin();
9122 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
9123 count += (int)intx_nodes.size() * 3 * sizeof( double );
9124 }
9125
9126 buff->check_space( count );
9127 PACK_INT( buff->buff_ptr, edges_to_send.size() );
9128 PACK_EH( buff->buff_ptr, &dum_remote_edges[0], dum_remote_edges.size() );
9129 for( Range::iterator eit = edges_to_send.begin(); eit != edges_to_send.end(); ++eit )
9130 {
9131 EntityHandle edge = *eit;
9132
9133 unsigned int indx = edges.find( edge ) - edges.begin();
9134 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
9135 PACK_INT( buff->buff_ptr, intx_nodes.size() );
9136
9137 result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), (double*)buff->buff_ptr );MB_CHK_SET_ERR( result, "Failed to get coords" );
9138 buff->buff_ptr += 3 * sizeof( double ) * intx_nodes.size();
9139 }
9140
9141
9142 buff->set_stored_size();
9143
9144
9145 result = send_buffer( *sit, localOwnedBuffs[ind], MB_MESG_TAGS_SIZE, sendReqs[3 * ind],
9146 recv_intx_reqs[3 * ind + 2], &dum_ack_buff, incoming );MB_CHK_SET_ERR( result, "Failed to send buffer" );
9147 }
9148
9149
9150 while( incoming )
9151 {
9152 MPI_Status status;
9153 int index_in_recv_requests;
9154 PRINT_DEBUG_WAITANY( recv_intx_reqs, MB_MESG_TAGS_SIZE, procConfig.proc_rank() );
9155 success = MPI_Waitany( 3 * buffProcs.size(), &recv_intx_reqs[0], &index_in_recv_requests, &status );
9156 if( MPI_SUCCESS != success )
9157 {
9158 MB_SET_ERR( MB_FAILURE, "Failed in waitany in ghost exchange" );
9159 }
9160
9161 ind = index_in_recv_requests / 3;
9162
9163 PRINT_DEBUG_RECD( status );
9164
9165
9166 incoming--;
9167
9168 bool done = false;
9169 result = recv_buffer( MB_MESG_TAGS_SIZE, status, remoteOwnedBuffs[ind],
9170 recv_intx_reqs[3 * ind + 1],
9171 recv_intx_reqs[3 * ind + 2],
9172
9173 incoming, localOwnedBuffs[ind],
9174 sendReqs[3 * ind + 1],
9175 sendReqs[3 * ind + 2],
9176 done );MB_CHK_SET_ERR( result, "Failed to resize recv buffer" );
9177 if( done )
9178 {
9179 Buffer* buff = remoteOwnedBuffs[ind];
9180 buff->reset_ptr( sizeof( int ) );
9181
9183
9184
9185 int num_edges;
9186
9187 UNPACK_INT( buff->buff_ptr, num_edges );
9188 std::vector< EntityHandle > rec_edges;
9189 rec_edges.resize( num_edges );
9190 UNPACK_EH( buff->buff_ptr, &rec_edges[0], num_edges );
9191 for( int i = 0; i < num_edges; i++ )
9192 {
9193 EntityHandle edge = rec_edges[i];
9194 unsigned int indx = edges.find( edge ) - edges.begin();
9195 std::vector< EntityHandle >& intx_nodes = *( extraNodesVec[indx] );
9196
9197 int nverts;
9198 UNPACK_INT( buff->buff_ptr, nverts );
9199 std::vector< double > pos_from_owner;
9200 pos_from_owner.resize( 3 * nverts );
9201 UNPACK_DBLS( buff->buff_ptr, &pos_from_owner[0], 3 * nverts );
9202 std::vector< double > current_positions( 3 * intx_nodes.size() );
9203 result = mbImpl->get_coords( &intx_nodes[0], intx_nodes.size(), ¤t_positions[0] );MB_CHK_SET_ERR( result, "Failed to get current positions" );
9204
9205 for( int k = 0; k < (int)intx_nodes.size(); k++ )
9206 {
9207 double* pk = ¤t_positions[3 * k];
9208
9209 bool found = false;
9210 for( int j = 0; j < nverts && !found; j++ )
9211 {
9212 double* pj = &pos_from_owner[3 * j];
9213 double dist2 = ( pk[0] - pj[0] ) * ( pk[0] - pj[0] ) + ( pk[1] - pj[1] ) * ( pk[1] - pj[1] ) +
9214 ( pk[2] - pj[2] ) * ( pk[2] - pj[2] );
9215 if( dist2 < tolerance )
9216 {
9217 pk[0] = pj[0];
9218 pk[1] = pj[1];
9219 pk[2] = pj[2];
9220 found = true;
9221 break;
9222 }
9223 }
9224 if( !found )
9225 {
9226 #ifndef NDEBUG
9227 std::cout << " pk:" << pk[0] << " " << pk[1] << " " << pk[2] << " not found \n";
9228 #endif
9229 result = MB_FAILURE;
9230 }
9231 }
9232
9233 result = mbImpl->set_coords( &intx_nodes[0], (int)intx_nodes.size(), ¤t_positions[0] );MB_CHK_SET_ERR( result, "Failed to set new current positions" );
9234 }
9235 }
9236 }
9237
9238
9239 if( myDebug->get_verbosity() == 5 )
9240 {
9241 success = MPI_Barrier( procConfig.proc_comm() );
9242 }
9243 else
9244 {
9245 MPI_Status status[3 * MAX_SHARING_PROCS];
9246 success = MPI_Waitall( 3 * buffProcs.size(), &sendReqs[0], status );
9247 }
9248 if( MPI_SUCCESS != success )
9249 {
9250 MB_SET_ERR( MB_FAILURE, "Failure in waitall in tag exchange" );
9251 }
9252
9253 myDebug->tprintf( 1, "Exiting settle_intersection_points" );
9254
9255 return MB_SUCCESS;
9256 }
9257
9258 ErrorCode ParallelComm::delete_entities( Range& to_delete )
9259 {
9260
9261
9262
9263 ErrorCode rval = MB_SUCCESS;
9264 unsigned char pstat;
9265 EntityHandle tmp_handles[MAX_SHARING_PROCS];
9266 int tmp_procs[MAX_SHARING_PROCS];
9267 unsigned int num_ps;
9268 TupleList ents_to_delete;
9269 ents_to_delete.initialize( 1, 0, 1, 0, to_delete.size() * ( MAX_SHARING_PROCS + 1 ) );
9270 ents_to_delete.enableWriteAccess();
9271 unsigned int i = 0;
9272 for( Range::iterator it = to_delete.begin(); it != to_delete.end(); ++it )
9273 {
9274 EntityHandle eh = *it;
9275
9276 rval = get_sharing_data( eh, tmp_procs, tmp_handles, pstat, num_ps );
9277 if( rval != MB_SUCCESS || num_ps == 0 ) continue;
9278
9279 for( unsigned int p = 0; p < num_ps; p++ )
9280 {
9281 ents_to_delete.vi_wr[i] = tmp_procs[p];
9282 ents_to_delete.vul_wr[i] = (unsigned long)tmp_handles[p];
9283 i++;
9284 ents_to_delete.inc_n();
9285 }
9286 }
9287
9288 gs_data::crystal_data* cd = this->procConfig.crystal_router();
9289
9290
9291 rval = cd->gs_transfer( 1, ents_to_delete, 0 );MB_CHK_SET_ERR( rval, "Error in tuple transfer" );
9292
9293
9294 unsigned int received = ents_to_delete.get_n();
9295 for( i = 0; i < received; i++ )
9296 {
9297
9298 unsigned long valrec = ents_to_delete.vul_rd[i];
9299 to_delete.insert( (EntityHandle)valrec );
9300 }
9301 rval = mbImpl->delete_entities( to_delete );MB_CHK_SET_ERR( rval, "Error in deleting actual entities" );
9302
9303 std::set< EntityHandle > good_ents;
9304 for( std::set< EntityHandle >::iterator sst = sharedEnts.begin(); sst != sharedEnts.end(); sst++ )
9305 {
9306 EntityHandle eh = *sst;
9307 int index = to_delete.index( eh );
9308 if( -1 == index ) good_ents.insert( eh );
9309 }
9310 sharedEnts = good_ents;
9311
9312
9313 return MB_SUCCESS;
9314 }
9315
9316 void ParallelComm::print_pstatus( unsigned char pstat, std::string& ostr )
9317 {
9318 std::ostringstream str;
9319 int num = 0;
9320 #define ppstat( a, b ) \
9321 { \
9322 if( pstat & ( a ) ) \
9323 { \
9324 if( num ) str << ", "; \
9325 str << ( b ); \
9326 num++; \
9327 } \
9328 }
9329
9330 ppstat( PSTATUS_NOT_OWNED, "NOT_OWNED" );
9331 ppstat( PSTATUS_SHARED, "SHARED" );
9332 ppstat( PSTATUS_MULTISHARED, "MULTISHARED" );
9333 ppstat( PSTATUS_INTERFACE, "INTERFACE" );
9334 ppstat( PSTATUS_GHOST, "GHOST" );
9335
9336 ostr = str.str();
9337 }
9338
9339 void ParallelComm::print_pstatus( unsigned char pstat )
9340 {
9341 std::string str;
9342 print_pstatus( pstat, str );
9343 std::cout << str.c_str() << std::endl;
9344 }
9345
9346 ErrorCode ParallelComm::correct_thin_ghost_layers()
9347 {
9348
9349
9350 std::vector< std::vector< SharedEntityData > > shents( buffProcs.size() ), send_data( buffProcs.size() );
9351
9352
9353
9354
9380
9381 ErrorCode result = MB_SUCCESS;
9382 int ent_procs[MAX_SHARING_PROCS + 1];
9383 EntityHandle handles[MAX_SHARING_PROCS + 1];
9384 int num_sharing;
9385 SharedEntityData tmp;
9386
9387 for( std::set< EntityHandle >::iterator i = sharedEnts.begin(); i != sharedEnts.end(); ++i )
9388 {
9389
9390 unsigned char pstat;
9391 result = get_sharing_data( *i, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" );
9392 if( !( pstat & PSTATUS_MULTISHARED ) ||
9393 num_sharing <= 2 )
9394 continue;
9395
9396
9397
9398 if( pstat & PSTATUS_NOT_OWNED ) continue;
9399 for( int j = 1; j < num_sharing; j++ )
9400 {
9401
9402 int send_to_proc = ent_procs[j];
9403 tmp.local = handles[j];
9404 int ind = get_buffers( send_to_proc );
9405 assert( -1 != ind );
9406 for( int k = 1; k < num_sharing; k++ )
9407 {
9408
9409 if( j == k ) continue;
9410 tmp.remote = handles[k];
9411 tmp.owner = ent_procs[k];
9412 send_data[ind].push_back( tmp );
9413 }
9414 }
9415 }
9416
9417 result = exchange_all_shared_handles( send_data, shents );MB_CHK_ERR( result );
9418
9419
9420 for( size_t i = 0; i < shents.size(); i++ )
9421 {
9422 std::vector< SharedEntityData >& shEnts = shents[i];
9423 for( size_t j = 0; j < shEnts.size(); j++ )
9424 {
9425 tmp = shEnts[j];
9426
9427
9428 EntityHandle eh = tmp.local;
9429 unsigned char pstat;
9430 result = get_sharing_data( eh, ent_procs, handles, pstat, num_sharing );MB_CHK_SET_ERR( result, "can't get sharing data" );
9431
9432
9433
9434 int proc_remote = tmp.owner;
9435 if( std::find( ent_procs, ent_procs + num_sharing, proc_remote ) == ent_procs + num_sharing )
9436 {
9437
9438 #ifndef NDEBUG
9439 if( myDebug->get_verbosity() == 3 )
9440 std::cout << "THIN GHOST: we did not find on proc " << rank() << " for shared ent " << eh
9441 << " the proc " << proc_remote << "\n";
9442 #endif
9443
9444 if( num_sharing >= MAX_SHARING_PROCS ) return MB_FAILURE;
9445 handles[num_sharing] = tmp.remote;
9446 handles[num_sharing + 1] = 0;
9447 ent_procs[num_sharing] = tmp.owner;
9448 ent_procs[num_sharing + 1] = -1;
9449 result = mbImpl->tag_set_data( sharedps_tag(), &eh, 1, ent_procs );MB_CHK_SET_ERR( result, "Failed to set sharedps tag data" );
9450 result = mbImpl->tag_set_data( sharedhs_tag(), &eh, 1, handles );MB_CHK_SET_ERR( result, "Failed to set sharedhs tag data" );
9451 if( 2 == num_sharing )
9452
9453 {
9454
9455
9456
9457 EntityHandle zero = 0;
9458 int no_proc = -1;
9459 result = mbImpl->tag_set_data( sharedp_tag(), &eh, 1, &no_proc );MB_CHK_SET_ERR( result, "Failed to set sharedp tag data" );
9460 result = mbImpl->tag_set_data( sharedh_tag(), &eh, 1, &zero );MB_CHK_SET_ERR( result, "Failed to set sharedh tag data" );
9461
9462
9463 pstat = pstat | PSTATUS_MULTISHARED;
9464 result = mbImpl->tag_set_data( pstatus_tag(), &eh, 1, &pstat );MB_CHK_SET_ERR( result, "Failed to set pstatus tag data" );
9465 }
9466 }
9467 }
9468 }
9469 return MB_SUCCESS;
9470 }
9471 }