Mesh Oriented datABase  (version 5.5.1)
An array-based unstructured mesh library
ParallelComm.hpp
Go to the documentation of this file.
1 /**
2  * MOAB, a Mesh-Oriented datABase, is a software component for creating,
3  * storing and accessing finite element mesh data.
4  *
5  * Copyright 2004 Sandia Corporation. Under the terms of Contract
6  * DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government
7  * retains certain rights in this software.
8  *
9  * This library is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU Lesser General Public
11  * License as published by the Free Software Foundation; either
12  * version 2.1 of the License, or (at your option) any later version.
13  *
14  */
15 
16 #ifndef MOAB_PARALLEL_COMM_HPP
17 #define MOAB_PARALLEL_COMM_HPP
18 
19 #include "moab/Forward.hpp"
20 #include "moab/Interface.hpp"
21 #include "moab/Range.hpp"
22 #include "moab/ProcConfig.hpp"
23 #include <map>
24 #include <set>
25 #include <vector>
26 #include <iostream>
27 #include <fstream>
28 #include <cassert>
29 #include <cstdlib>
30 #include <cmath>
31 #include "moab/TupleList.hpp"
32 
33 namespace moab
34 {
35 
36 class SequenceManager;
37 class Error;
38 template < typename KeyType, typename ValType, ValType NullVal >
39 class RangeMap;
40 typedef RangeMap< EntityHandle, EntityHandle, 0 > HandleMap;
41 class ParallelMergeMesh;
42 class DebugOutput;
43 class SharedSetData;
44 
45 #define MAX_SHARING_PROCS 64
46 
47 /**
48  * \brief Parallel communications in MOAB
49  * \author Tim Tautges
50  *
51  * This class implements methods to communicate mesh between processors
52  *
53  */
54 class ParallelComm
55 {
56  public:
57  friend class ParallelMergeMesh;
58 
59  // ==================================
60  // \section CONSTRUCTORS/DESTRUCTORS/PCOMM MANAGEMENT
61  // ==================================
62 
63  //! constructor
64  ParallelComm( Interface* impl, MPI_Comm comm, int* pcomm_id_out = 0 );
65 
66  //! constructor taking packed buffer, for testing
67  ParallelComm( Interface* impl, std::vector< unsigned char >& tmp_buff, MPI_Comm comm, int* pcomm_id_out = 0 );
68 
69  //! Get ID used to reference this PCOMM instance
70  int get_id() const
71  {
72  return pcommID;
73  }
74 
75  //! get the indexed pcomm object from the interface
76  static ParallelComm* get_pcomm( Interface* impl, const int index );
77 
78  //! Get ParallelComm instance associated with partition handle
79  //! Will create ParallelComm instance if a) one does not already
80  //! exist and b) a valid value for MPI_Comm is passed.
81  static ParallelComm* get_pcomm( Interface* impl, EntityHandle partitioning, const MPI_Comm* comm = 0 );
82 
83  static ErrorCode get_all_pcomm( Interface* impl, std::vector< ParallelComm* >& list );
84 
85  //! destructor
86  ~ParallelComm();
87 
88  static unsigned char PROC_SHARED, PROC_OWNER;
89 
90  // ==================================
91  // \section GLOBAL IDS
92  // ==================================
93 
94  //! assign a global id space, for largest-dimension or all entities (and
95  //! in either case for vertices too)
96  //!\param owned_only If true, do not get global IDs for non-owned entities
97  //! from remote processors.
98  ErrorCode assign_global_ids( EntityHandle this_set,
99  const int dimension,
100  const int start_id = 1,
101  const bool largest_dim_only = true,
102  const bool parallel = true,
103  const bool owned_only = false );
104 
105  //! assign a global id space, for largest-dimension or all entities (and
106  //! in either case for vertices too)
107  ErrorCode assign_global_ids( Range entities[],
108  const int dimension,
109  const int start_id,
110  const bool parallel,
111  const bool owned_only );
112 
113  //! check for global ids; based only on tag handle being there or not;
114  //! if it's not there, create them for the specified dimensions
115  //!\param owned_only If true, do not get global IDs for non-owned entities
116  //! from remote processors.
117  ErrorCode check_global_ids( EntityHandle this_set,
118  const int dimension,
119  const int start_id = 1,
120  const bool largest_dim_only = true,
121  const bool parallel = true,
122  const bool owned_only = false );
123 
124  // ==================================
125  // \section HIGH-LEVEL COMMUNICATION (send/recv/bcast/scatter ents, exchange tags)
126  // ==================================
127 
128  /** \brief send entities to another processor, optionally waiting until it's done
129  *
130  * Send entities to another processor, with adjs, sets, and tags.
131  * If store_remote_handles is true, this call receives back handles assigned to
132  * entities sent to destination processor and stores them in sharedh_tag or
133  * sharedhs_tag.
134  * \param to_proc Destination processor
135  * \param orig_ents Entities requested to send
136  * \param adjs If true, send adjacencies for equiv entities (currently unsupported)
137  * \param tags If true, send tag values for all tags assigned to entities
138  * \param store_remote_handles If true, also recv message with handles on destination processor
139  * (currently unsupported) \param final_ents Range containing all entities sent \param incoming
140  * keep track if any messages are coming to this processor (newly added) \param wait_all If
141  * true, wait until all messages received/sent complete
142  */
143  ErrorCode send_entities( const int to_proc,
144  Range& orig_ents,
145  const bool adjs,
146  const bool tags,
147  const bool store_remote_handles,
148  const bool is_iface,
149  Range& final_ents,
150  int& incoming1,
151  int& incoming2, // newly added
152  TupleList& entprocs, // newly added
153  std::vector< MPI_Request >& recv_remoteh_reqs, // newly added
154  bool wait_all = true );
155 
156  ErrorCode send_entities( std::vector< unsigned int >& send_procs,
157  std::vector< Range* >& send_ents,
158  int& incoming1,
159  int& incoming2,
160  const bool store_remote_handles );
161 
162  /** \brief Receive entities from another processor, optionally waiting until it's done
163  *
164  * Receive entities from another processor, with adjs, sets, and tags.
165  * If store_remote_handles is true, this call sends back handles assigned to
166  * the entities received.
167  * \param from_proc Source processor
168  * \param store_remote_handles If true, send message with new entity handles to source processor
169  * (currently unsupported) \param final_ents Range containing all entities received \param
170  * incoming keep track if any messages are coming to this processor (newly added) \param
171  * wait_all If true, wait until all messages received/sent complete
172  */
173  ErrorCode recv_entities( const int from_proc,
174  const bool store_remote_handles,
175  const bool is_iface,
176  Range& final_ents,
177  int& incomming1,
178  int& incoming2,
179  std::vector< std::vector< EntityHandle > >& L1hloc,
180  std::vector< std::vector< EntityHandle > >& L1hrem,
181  std::vector< std::vector< int > >& L1p,
182  std::vector< EntityHandle >& L2hloc,
183  std::vector< EntityHandle >& L2hrem,
184  std::vector< unsigned int >& L2p,
185  std::vector< MPI_Request >& recv_remoteh_reqs,
186  bool wait_all = true );
187 
188  ErrorCode recv_entities( std::set< unsigned int >& recv_procs,
189  int incoming1,
190  int incoming2,
191  const bool store_remote_handles,
192  const bool migrate = false );
193 
194  /** \brief Receive messages from another processor in while loop
195  *
196  * Receive messages from another processor.
197  * \param from_proc Source processor
198  * \param store_remote_handles If true, send message with new entity handles to source processor
199  * (currently unsupported) \param final_ents Range containing all entities received \param
200  * incoming keep track if any messages are coming to this processor (newly added)
201  */
202  ErrorCode recv_messages( const int from_proc,
203  const bool store_remote_handles,
204  const bool is_iface,
205  Range& final_ents,
206  int& incoming1,
207  int& incoming2,
208  std::vector< std::vector< EntityHandle > >& L1hloc,
209  std::vector< std::vector< EntityHandle > >& L1hrem,
210  std::vector< std::vector< int > >& L1p,
211  std::vector< EntityHandle >& L2hloc,
212  std::vector< EntityHandle >& L2hrem,
213  std::vector< unsigned int >& L2p,
214  std::vector< MPI_Request >& recv_remoteh_reqs );
215 
216  ErrorCode recv_remote_handle_messages( const int from_proc,
217  int& incoming2,
218  std::vector< EntityHandle >& L2hloc,
219  std::vector< EntityHandle >& L2hrem,
220  std::vector< unsigned int >& L2p,
221  std::vector< MPI_Request >& recv_remoteh_reqs );
222 
223  /** \brief Exchange ghost cells with neighboring procs
224  * Neighboring processors are those sharing an interface
225  * with this processor. All entities of dimension ghost_dim
226  * within num_layers of interface, measured going through bridge_dim,
227  * are exchanged. See MeshTopoUtil::get_bridge_adjacencies for description
228  * of bridge adjacencies. If wait_all is false and store_remote_handles
229  * is true, MPI_Request objects are available in the sendReqs[2*MAX_SHARING_PROCS]
230  * member array, with inactive requests marked as MPI_REQUEST_NULL. If
231  * store_remote_handles or wait_all is false, this function returns after
232  * all entities have been received and processed.
233  * \param ghost_dim Dimension of ghost entities to be exchanged
234  * \param bridge_dim Dimension of entities used to measure layers from interface
235  * \param num_layers Number of layers of ghosts requested
236  * \param addl_ents Dimension of additional adjacent entities to exchange with ghosts, 0 if none
237  * \param store_remote_handles If true, send message with new entity handles to source processor
238  * \param wait_all If true, function does not return until all send buffers
239  * are cleared.
240  */
241 
242  ErrorCode exchange_ghost_cells( int ghost_dim,
243  int bridge_dim,
244  int num_layers,
245  int addl_ents,
246  bool store_remote_handles,
247  bool wait_all = true,
248  EntityHandle* file_set = NULL );
249 
250  /** \brief Static version of exchange_ghost_cells, exchanging info through
251  * buffers rather than messages
252  */
253  static ErrorCode exchange_ghost_cells( ParallelComm** pc,
254  unsigned int num_procs,
255  int ghost_dim,
256  int bridge_dim,
257  int num_layers,
258  int addl_ents,
259  bool store_remote_handles,
260  EntityHandle* file_sets = NULL );
261 
262  /** \brief Post "MPI_Irecv" before meshing
263  * \param exchange_procs processor vector exchanged
264  */
265  ErrorCode post_irecv( std::vector< unsigned int >& exchange_procs );
266 
267  ErrorCode post_irecv( std::vector< unsigned int >& shared_procs, std::set< unsigned int >& recv_procs );
268 
269  /** \brief Exchange owned mesh for input mesh entities and sets
270  * This function should be called collectively over the communicator for this ParallelComm.
271  * If this version is called, all shared exchanged entities should have a value for this
272  * tag (or the tag should have a default value).
273  * \param exchange_procs processor vector exchanged
274  * \param exchange_ents exchanged entities for each processors
275  * \param migrate if the owner if entities are changed or not
276  */
277  ErrorCode exchange_owned_meshs( std::vector< unsigned int >& exchange_procs,
278  std::vector< Range* >& exchange_ents,
279  std::vector< MPI_Request >& recv_ent_reqs,
280  std::vector< MPI_Request >& recv_remoteh_reqs,
281  bool store_remote_handles,
282  bool wait_all = true,
283  bool migrate = false,
284  int dim = 0 );
285 
286  /** \brief Exchange owned mesh for input mesh entities and sets
287  * This function is called twice by exchange_owned_meshs to exchange entities before sets
288  * \param migrate if the owner if entities are changed or not
289  */
290  ErrorCode exchange_owned_mesh( std::vector< unsigned int >& exchange_procs,
291  std::vector< Range* >& exchange_ents,
292  std::vector< MPI_Request >& recv_ent_reqs,
293  std::vector< MPI_Request >& recv_remoteh_reqs,
294  const bool recv_posted,
295  bool store_remote_handles,
296  bool wait_all,
297  bool migrate = false );
298 
299  /** \brief Exchange tags for all shared and ghosted entities
300  * This function should be called collectively over the communicator for this ParallelComm.
301  * If this version is called, all ghosted/shared entities should have a value for this
302  * tag (or the tag should have a default value). If the entities vector is empty, all shared
303  * entities participate in the exchange. If a proc has no owned entities this function must
304  * still be called since it is collective. \param src_tags Vector of tag handles to be exchanged
305  * \param dst_tags Tag handles to store the tags on the non-owning procs
306  * \param entities Entities for which tags are exchanged
307  */
308  ErrorCode exchange_tags( const std::vector< Tag >& src_tags,
309  const std::vector< Tag >& dst_tags,
310  const Range& entities );
311 
312  /** \brief Exchange tags for all shared and ghosted entities
313  * This function should be called collectively over the communicator for this ParallelComm.
314  * If the entities vector is empty, all shared entities
315  * participate in the exchange. If a proc has no owned entities this function must still be
316  * called since it is collective. \param tag_name Name of tag to be exchanged \param entities
317  * Entities for which tags are exchanged
318  */
319  ErrorCode exchange_tags( const char* tag_name, const Range& entities );
320 
321  /** \brief Exchange tags for all shared and ghosted entities
322  * This function should be called collectively over the communicator for this ParallelComm.
323  * If the entities vector is empty, all shared entities
324  * participate in the exchange. If a proc has no owned entities this function must still be
325  * called since it is collective. \param tagh Handle of tag to be exchanged \param entities
326  * Entities for which tags are exchanged
327  */
328  ErrorCode exchange_tags( Tag tagh, const Range& entities );
329 
330  /** \brief Perform data reduction operation for all shared and ghosted entities
331  * This function should be called collectively over the communicator for this ParallelComm.
332  * If this version is called, all ghosted/shared entities should have a value for this
333  * tag (or the tag should have a default value). Operation is any MPI_Op, with result stored
334  * in destination tag.
335  * \param src_tags Vector of tag handles to be reduced
336  * \param dst_tags Vector of tag handles in which the answer will be stored
337  * \param mpi_op Operation type
338  * \param entities Entities on which reduction will be made; if empty, operates on all shared
339  * entities
340  */
341  ErrorCode reduce_tags( const std::vector< Tag >& src_tags,
342  const std::vector< Tag >& dst_tags,
343  const MPI_Op mpi_op,
344  const Range& entities );
345 
346  /** \brief Perform data reduction operation for all shared and ghosted entities
347  * Same as std::vector variant except for one tag specified by name
348  * \param tag_name Name of tag to be reduced
349  * \param mpi_op Operation type
350  * \param entities Entities on which reduction will be made; if empty, operates on all shared
351  * entities
352  */
353  ErrorCode reduce_tags( const char* tag_name, const MPI_Op mpi_op, const Range& entities );
354 
355  /** \brief Perform data reduction operation for all shared and ghosted entities
356  * Same as std::vector variant except for one tag specified by handle
357  * \param tag_name Name of tag to be reduced
358  * \param mpi_op Operation type
359  * \param entities Entities on which reduction will be made; if empty, operates on all shared
360  * entities
361  */
362  ErrorCode reduce_tags( Tag tag_handle, const MPI_Op mpi_op, const Range& entities );
363 
364  /** \brief Broadcast all entities resident on from_proc to other processors
365  * This function assumes remote handles are *not* being stored, since (usually)
366  * every processor will know about the whole mesh.
367  * \param from_proc Processor having the mesh to be broadcast
368  * \param entities On return, the entities sent or received in this call
369  * \param adjacencies If true, adjacencies are sent for equiv entities (currently unsupported)
370  * \param tags If true, all non-default-valued tags are sent for sent entities
371  */
372  ErrorCode broadcast_entities( const int from_proc,
373  Range& entities,
374  const bool adjacencies = false,
375  const bool tags = true );
376 
377  /** \brief Scatter entities on from_proc to other processors
378  * This function assumes remote handles are *not* being stored, since (usually)
379  * every processor will know about the whole mesh.
380  * \param from_proc Processor having the mesh to be broadcast
381  * \param entities On return, the entities sent or received in this call
382  * \param adjacencies If true, adjacencies are sent for equiv entities (currently unsupported)
383  * \param tags If true, all non-default-valued tags are sent for sent entities
384  */
385  ErrorCode scatter_entities( const int from_proc,
386  std::vector< Range >& entities,
387  const bool adjacencies = false,
388  const bool tags = true );
389 
390  /////////////////////////////////////////////////////////////////////////////////
391  // Send and Receive routines for a sequence of entities: use case UMR
392  /////////////////////////////////////////////////////////////////////////////////
393 
394  /** \brief Send and receives data from a set of processors
395  */
396  ErrorCode send_recv_entities( std::vector< int >& send_procs,
397  std::vector< std::vector< int > >& msgsizes,
398  std::vector< std::vector< EntityHandle > >& senddata,
399  std::vector< std::vector< EntityHandle > >& recvdata );
400 
401  ErrorCode update_remote_data( EntityHandle entity,
402  std::vector< int >& procs,
403  std::vector< EntityHandle >& handles );
404 
405  ErrorCode get_remote_handles( EntityHandle* local_vec, EntityHandle* rem_vec, int num_ents, int to_proc );
406 
407  /////////////////////////////////////////////////////////////////////////////////
408 
409  // ==================================
410  // \section INITIALIZATION OF PARALLEL DATA (resolve_shared_ents, etc.)
411  // ==================================
412 
413  /** \brief Resolve shared entities between processors
414  *
415  * Resolve shared entities between processors for entities in proc_ents,
416  * by comparing global id tag values on vertices on skin of elements in
417  * proc_ents. Shared entities are assigned a tag that's either
418  * PARALLEL_SHARED_PROC_TAG_NAME, which is 1 integer in length, or
419  * PARALLEL_SHARED_PROCS_TAG_NAME, whose length depends on the maximum
420  * number of sharing processors. Values in these tags denote the ranks
421  * of sharing processors, and the list ends with the value -1.
422  *
423  * If shared_dim is input as -1 or not input, a value one less than the
424  * maximum dimension of entities in proc_ents is used.
425  *
426  * \param proc_ents Entities for which to resolve shared entities
427  * \param shared_dim Maximum dimension of shared entities to look for
428  */
429  ErrorCode resolve_shared_ents( EntityHandle this_set,
430  Range& proc_ents,
431  int resolve_dim = -1,
432  int shared_dim = -1,
433  Range* skin_ents = NULL,
434  const Tag* id_tag = 0 );
435 
436  /** \brief Resolve shared entities between processors
437  *
438  * Same as resolve_shared_ents(Range&), except works for
439  * all entities in instance of dimension dim.
440  *
441  * If shared_dim is input as -1 or not input, a value one less than the
442  * maximum dimension of entities is used.
443 
444  * \param dim Dimension of entities in the partition
445  * \param shared_dim Maximum dimension of shared entities to look for
446  */
447  ErrorCode resolve_shared_ents( EntityHandle this_set,
448  int resolve_dim = 3,
449  int shared_dim = -1,
450  const Tag* id_tag = 0 );
451 
452  static ErrorCode resolve_shared_ents( ParallelComm** pc,
453  const unsigned int np,
454  EntityHandle this_set,
455  const int to_dim );
456 
457  /** Remove shared sets.
458  *
459  * Generates list of candidate sets using from those (directly)
460  * contained in passed set and passes them to the other version
461  * of \c resolve_shared_sets.
462  *\param this_set Set directly containing candidate sets (e.g. file set)
463  *\param id_tag Tag containing global IDs for entity sets.
464  */
465 
466  ErrorCode resolve_shared_sets( EntityHandle this_set, const Tag* id_tag = 0 );
467 
468  /** Remove shared sets.
469  *
470  * Use values of id_tag to match sets across processes and populate
471  * sharing data for sets.
472  *\param candidate_sets Sets to consider as potentially shared.
473  *\param id_tag Tag containing global IDs for entity sets.
474  */
475  ErrorCode resolve_shared_sets( Range& candidate_sets, Tag id_tag );
476 
477  /** extend shared sets with ghost entities
478  * After ghosting, ghost entities do not have yet information about
479  * the material set, partition set, Neumann or Dirichlet set they could
480  * belong to
481  * This method will assign ghosted entities to the those special entity sets
482  * In some case we might even have to create those sets, if they do not exist yet on
483  * the local processor
484  *
485  * The special entity sets all have an unique identifier, in a form of an integer
486  * tag to the set.
487  * The shared sets data is not used, because we do not use the geometry sets, as they are
488  * not uniquely identified
489  *
490  *
491  * \param file_set : file set used per application
492  *
493  */
494  ErrorCode augment_default_sets_with_ghosts( EntityHandle file_set );
495  // ==================================
496  // \section GET PARALLEL DATA (shared/owned/iface entities, etc.)
497  // ==================================
498 
499  /** \brief Get parallel status of an entity
500  * Returns the parallel status of an entity
501  *
502  * \param entity The entity being queried
503  * \param pstatus_val Parallel status of the entity
504  */
505  ErrorCode get_pstatus( EntityHandle entity, unsigned char& pstatus_val );
506 
507  /** \brief Get entities with the given pstatus bit(s) set
508  * Returns any entities whose pstatus tag value v satisfies (v & pstatus_val)
509  *
510  * \param dim Dimension of entities to be returned, or -1 if any
511  * \param pstatus_val pstatus value of desired entities
512  * \param pstatus_ents Entities returned from function
513  */
514  ErrorCode get_pstatus_entities( int dim, unsigned char pstatus_val, Range& pstatus_ents );
515 
516  /** \brief Return the rank of the entity owner
517  */
518  ErrorCode get_owner( EntityHandle entity, int& owner );
519 
520  /** \brief Return the owner processor and handle of a given entity
521  */
522  ErrorCode get_owner_handle( EntityHandle entity, int& owner, EntityHandle& handle );
523 
524  /** \brief Get the shared processors/handles for an entity
525  * Get the shared processors/handles for an entity. Arrays must
526  * be large enough to receive data for all sharing procs. Does *not* include
527  * this proc if only shared with one other proc.
528  * \param entity Entity being queried
529  * \param ps Pointer to sharing proc data
530  * \param hs Pointer to shared proc handle data
531  * \param pstat Reference to pstatus data returned from this function
532  */
533  ErrorCode get_sharing_data( const EntityHandle entity,
534  int* ps,
535  EntityHandle* hs,
536  unsigned char& pstat,
537  unsigned int& num_ps );
538 
539  /** \brief Get the shared processors/handles for an entity
540  * Same as other version but with int num_ps
541  * \param entity Entity being queried
542  * \param ps Pointer to sharing proc data
543  * \param hs Pointer to shared proc handle data
544  * \param pstat Reference to pstatus data returned from this function
545  */
546  ErrorCode get_sharing_data( const EntityHandle entity,
547  int* ps,
548  EntityHandle* hs,
549  unsigned char& pstat,
550  int& num_ps );
551 
552  /** \brief Get the intersection or union of all sharing processors
553  * Get the intersection or union of all sharing processors. Processor set
554  * is cleared as part of this function.
555  * \param entities Entity list ptr
556  * \param num_entities Number of entities
557  * \param procs Processors returned
558  * \param op Either Interface::UNION or Interface::INTERSECT
559  */
560  ErrorCode get_sharing_data( const EntityHandle* entities,
561  int num_entities,
562  std::set< int >& procs,
563  int op = Interface::INTERSECT );
564 
565  /** \brief Get the intersection or union of all sharing processors
566  * Same as previous variant but with range as input
567  */
568  ErrorCode get_sharing_data( const Range& entities, std::set< int >& procs, int op = Interface::INTERSECT );
569 
570  /** \brief Get shared entities of specified dimension
571  * If other_proc is -1, any shared entities are returned. If dim is -1,
572  * entities of all dimensions on interface are returned.
573  * \param other_proc Rank of processor for which interface entities are requested
574  * \param shared_ents Entities returned from function
575  * \param dim Dimension of interface entities requested
576  * \param iface If true, return only entities on the interface
577  * \param owned_filter If true, return only owned shared entities
578  */
579  ErrorCode get_shared_entities( int other_proc,
580  Range& shared_ents,
581  int dim = -1,
582  const bool iface = false,
583  const bool owned_filter = false );
584  /*
585  //! return partition sets; if tag_name is input, gets sets with
586  //! that tag name, otherwise uses PARALLEL_PARTITION tag
587  ErrorCode get_partition_sets(EntityHandle this_set,
588  Range &part_sets,
589  const char *tag_name = NULL);
590  */
591  //! get processors with which this processor shares an interface
592  ErrorCode get_interface_procs( std::set< unsigned int >& iface_procs, const bool get_buffs = false );
593 
594  //! get processors with which this processor communicates
595  ErrorCode get_comm_procs( std::set< unsigned int >& procs );
596 
597  // ==================================
598  // \section SHARED SETS
599  // ==================================
600 
601  //! Get array of process IDs sharing a set. Returns zero
602  //! and passes back NULL if set is not shared.
603  ErrorCode get_entityset_procs( EntityHandle entity_set, std::vector< unsigned >& ranks ) const;
604 
605  //! Get rank of the owner of a shared set.
606  //! Returns this proc if set is not shared.
607  //! Optionally returns handle on owning process for shared set.
608  ErrorCode get_entityset_owner( EntityHandle entity_set,
609  unsigned& owner_rank,
610  EntityHandle* remote_handle = 0 ) const;
611 
612  //! Given set owner and handle on owner, find local set handle
613  ErrorCode get_entityset_local_handle( unsigned owning_rank,
614  EntityHandle remote_handle,
615  EntityHandle& local_handle ) const;
616 
617  //! Get all shared sets
618  ErrorCode get_shared_sets( Range& result ) const;
619 
620  //! Get ranks of all processes that own at least one set that is
621  //! shared with this process. Will include the rank of this process
622  //! if this process owns any shared set.
623  ErrorCode get_entityset_owners( std::vector< unsigned >& ranks ) const;
624 
625  //! Get shared sets owned by process with specified rank.
626  ErrorCode get_owned_sets( unsigned owning_rank, Range& sets_out ) const;
627 
628  // ==================================
629  // \section LOW-LEVEL DATA (tags, sets on interface/partition, etc.)
630  // ==================================
631 
632  //! Get proc config for this communication object
633  const ProcConfig& proc_config() const
634  {
635  return procConfig;
636  }
637 
638  //! Get proc config for this communication object
639  ProcConfig& proc_config()
640  {
641  return procConfig;
642  }
643 
644  unsigned rank() const
645  {
646  return proc_config().proc_rank();
647  }
648  unsigned size() const
649  {
650  return proc_config().proc_size();
651  }
652  MPI_Comm comm() const
653  {
654  return proc_config().proc_comm();
655  }
656 
657  //! return the tags used to indicate shared procs and handles
658  ErrorCode get_shared_proc_tags( Tag& sharedp_tag,
659  Tag& sharedps_tag,
660  Tag& sharedh_tag,
661  Tag& sharedhs_tag,
662  Tag& pstatus_tag );
663 
664  //! return partition, interface set ranges
665  Range& partition_sets()
666  {
667  return partitionSets;
668  }
669  const Range& partition_sets() const
670  {
671  return partitionSets;
672  }
673  Range& interface_sets()
674  {
675  return interfaceSets;
676  }
677  const Range& interface_sets() const
678  {
679  return interfaceSets;
680  }
681 
682  //! return sharedp tag
683  Tag sharedp_tag();
684 
685  //! return sharedps tag
686  Tag sharedps_tag();
687 
688  //! return sharedh tag
689  Tag sharedh_tag();
690 
691  //! return sharedhs tag
692  Tag sharedhs_tag();
693 
694  //! return pstatus tag
695  Tag pstatus_tag();
696 
697  //! return pcomm tag; static because might not have a pcomm before going
698  //! to look for one on the interface
699  static Tag pcomm_tag( Interface* impl, bool create_if_missing = true );
700 
701  //! return partitions set tag
702  Tag partition_tag();
703  Tag part_tag()
704  {
705  return partition_tag();
706  }
707 
708  // ==================================
709  // \section DEBUGGING AIDS
710  // ==================================
711 
712  //! print contents of pstatus value in human-readable form
713  void print_pstatus( unsigned char pstat, std::string& ostr );
714 
715  //! print contents of pstatus value in human-readable form to std::cut
716  void print_pstatus( unsigned char pstat );
717 
718  // ==================================
719  // \section IMESHP-RELATED FUNCTIONS
720  // ==================================
721 
722  //! return all the entities in parts owned locally
723  ErrorCode get_part_entities( Range& ents, int dim = -1 );
724 
725  EntityHandle get_partitioning() const
726  {
727  return partitioningSet;
728  }
729  ErrorCode set_partitioning( EntityHandle h );
730  ErrorCode get_global_part_count( int& count_out ) const;
731  ErrorCode get_part_owner( int part_id, int& owner_out ) const;
732  ErrorCode get_part_id( EntityHandle part, int& id_out ) const;
733  ErrorCode get_part_handle( int id, EntityHandle& handle_out ) const;
734  ErrorCode create_part( EntityHandle& part_out );
735  ErrorCode destroy_part( EntityHandle part );
736  ErrorCode collective_sync_partition();
737  ErrorCode get_part_neighbor_ids( EntityHandle part, int neighbors_out[MAX_SHARING_PROCS], int& num_neighbors_out );
738  ErrorCode get_interface_sets( EntityHandle part, Range& iface_sets_out, int* adj_part_id = 0 );
739  ErrorCode get_owning_part( EntityHandle entity, int& owning_part_id_out, EntityHandle* owning_handle = 0 );
740  ErrorCode get_sharing_parts( EntityHandle entity,
741  int part_ids_out[MAX_SHARING_PROCS],
742  int& num_part_ids_out,
743  EntityHandle remote_handles[MAX_SHARING_PROCS] = 0 );
744 
745  /** Filter the entities by pstatus tag.
746  * op is one of PSTATUS_ AND, OR, NOT; an entity is output if:
747  * AND: all bits set in pstatus_val are also set on entity
748  * OR: any bits set in pstatus_val also set on entity
749  * NOT: any bits set in pstatus_val are not set on entity
750  *
751  * Results returned in input list, unless result_ents is passed in non-null,
752  * in which case results are returned in result_ents.
753  *
754  * If ents is passed in empty, filter is done on shared entities in this
755  * pcomm instance, i.e. contents of sharedEnts.
756  *
757  *\param ents Input entities to filter
758  *\param pstatus_val pstatus value to which entities are compared
759  *\param op Bitwise operation performed between pstatus values
760  *\param to_proc If non-negative and PSTATUS_SHARED is set on pstatus_val,
761  * only entities shared with to_proc are returned
762  *\param result_ents If non-null, results of filter are put in the
763  * pointed-to range
764  */
765  ErrorCode filter_pstatus( Range& ents,
766  const unsigned char pstatus_val,
767  const unsigned char op,
768  int to_proc = -1,
769  Range* returned_ents = NULL );
770 
771  /** \brief Get entities on interfaces shared with another proc
772  *
773  * \param other_proc Other proc sharing the interface
774  * \param dim Dimension of entities to return, -1 if all dims
775  * \param iface_ents Returned entities
776  */
777  ErrorCode get_iface_entities( int other_proc, int dim, Range& iface_ents );
778 
779  Interface* get_moab() const
780  {
781  return mbImpl;
782  }
783 
784  ErrorCode clean_shared_tags( std::vector< Range* >& exchange_ents );
785 
786  class Buffer
787  {
788  public:
789  unsigned char* mem_ptr;
790  unsigned char* buff_ptr;
791  unsigned int alloc_size;
792 
793  Buffer( unsigned int sz = 0 );
794  Buffer( const Buffer& );
795  ~Buffer();
796  void reset_buffer( size_t buff_pos = 0 )
797  {
798  reset_ptr( buff_pos );
799  reserve( INITIAL_BUFF_SIZE );
800  }
801  void reset_ptr( size_t buff_pos = 0 )
802  {
803  assert( ( !mem_ptr && !buff_pos ) || ( alloc_size >= buff_pos ) );
804  buff_ptr = mem_ptr + buff_pos;
805  }
806  inline void reserve( unsigned int new_size );
807  void set_stored_size()
808  {
809  *( (int*)mem_ptr ) = (int)( buff_ptr - mem_ptr );
810  }
811  int get_stored_size()
812  {
813  return *( (int*)mem_ptr );
814  }
815  int get_current_size()
816  {
817  return (int)( buff_ptr - mem_ptr );
818  }
819 
820  void check_space( unsigned int addl_space );
821  };
822 
823  //! public 'cuz we want to unit test these externally
824  ErrorCode pack_buffer( Range& orig_ents,
825  const bool adjacencies,
826  const bool tags,
827  const bool store_remote_handles,
828  const int to_proc,
829  Buffer* buff,
830  TupleList* entprocs = NULL,
831  Range* allsent = NULL );
832 
833  ErrorCode unpack_buffer( unsigned char* buff_ptr,
834  const bool store_remote_handles,
835  const int from_proc,
836  const int ind,
837  std::vector< std::vector< EntityHandle > >& L1hloc,
838  std::vector< std::vector< EntityHandle > >& L1hrem,
839  std::vector< std::vector< int > >& L1p,
840  std::vector< EntityHandle >& L2hloc,
841  std::vector< EntityHandle >& L2hrem,
842  std::vector< unsigned int >& L2p,
843  std::vector< EntityHandle >& new_ents,
844  const bool created_iface = false );
845 
846  ErrorCode pack_entities( Range& entities,
847  Buffer* buff,
848  const bool store_remote_handles,
849  const int to_proc,
850  const bool is_iface,
851  TupleList* entprocs = NULL,
852  Range* allsent = NULL );
853 
854  //! unpack entities in buff_ptr
855  ErrorCode unpack_entities( unsigned char*& buff_ptr,
856  const bool store_remote_handles,
857  const int from_ind,
858  const bool is_iface,
859  std::vector< std::vector< EntityHandle > >& L1hloc,
860  std::vector< std::vector< EntityHandle > >& L1hrem,
861  std::vector< std::vector< int > >& L1p,
862  std::vector< EntityHandle >& L2hloc,
863  std::vector< EntityHandle >& L2hrem,
864  std::vector< unsigned int >& L2p,
865  std::vector< EntityHandle >& new_ents,
866  const bool created_iface = false );
867 
868  //! Call exchange_all_shared_handles, then compare the results with tag data
869  //! on local shared entities.
870  ErrorCode check_all_shared_handles( bool print_em = false );
871 
872  static ErrorCode check_all_shared_handles( ParallelComm** pcs, int num_pcs );
873 
874  struct SharedEntityData
875  {
876  EntityHandle local;
877  EntityHandle remote;
878  EntityID owner;
879  };
880 
881  ErrorCode pack_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data );
882 
883  // check consistency of sharedEnts against their tags and their
884  // vertices' tags
885  ErrorCode check_local_shared();
886 
887  // check contents of communicated shared entity data against tags
888  ErrorCode check_my_shared_handles( std::vector< std::vector< SharedEntityData > >& shents,
889  const char* prefix = NULL );
890 
891  //! set rank for this pcomm; USED FOR TESTING ONLY!
892  void set_rank( unsigned int r );
893 
894  //! set rank for this pcomm; USED FOR TESTING ONLY!
895  void set_size( unsigned int r );
896 
897  //! get (and possibly allocate) buffers for messages to/from to_proc; returns
898  //! index of to_proc in buffProcs vector; if is_new is non-NULL, sets to
899  //! whether new buffer was allocated
900  //! PUBLIC ONLY FOR TESTING!
901  int get_buffers( int to_proc, bool* is_new = NULL );
902 
903  //! get buff processor vector
904  const std::vector< unsigned int >& buff_procs() const;
905 
906  /* \brief Unpack message with remote handles
907  * PUBLIC ONLY FOR TESTING!
908  */
909  ErrorCode unpack_remote_handles( unsigned int from_proc,
910  unsigned char*& buff_ptr,
911  std::vector< EntityHandle >& L2hloc,
912  std::vector< EntityHandle >& L2hrem,
913  std::vector< unsigned int >& L2p );
914 
915  /* \brief Pack message with remote handles
916  * PUBLIC ONLY FOR TESTING!
917  */
918  ErrorCode pack_remote_handles( std::vector< EntityHandle >& L1hloc,
919  std::vector< EntityHandle >& L1hrem,
920  std::vector< int >& procs,
921  unsigned int to_proc,
922  Buffer* buff );
923 
924  // each iterate in proc_nvecs contains a set of procs and the entities *possibly*
925  // on the interface between those procs; this function makes sets for each,
926  // and tags the set with the procs sharing it; interface sets are optionally
927  // returned; NOTE: a subsequent step is used to verify entities on the interface
928  // and remove them if they're not shared
929  ErrorCode create_interface_sets( std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs );
930 
931  // do the same but working straight from sharedEnts
932  ErrorCode create_interface_sets( EntityHandle this_set, int resolve_dim, int shared_dim );
933 
934  ErrorCode tag_shared_verts( TupleList& shared_ents,
935  std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
936  Range& proc_verts,
937  unsigned int i_extra = 1 );
938 
939  ErrorCode list_entities( const EntityHandle* ents, int num_ents );
940 
941  ErrorCode list_entities( const Range& ents );
942 
943  void set_send_request( int n_request ); // set send request array
944 
945  void set_recv_request( int n_request ); // set recv request array
946 
947  //! reset message buffers to their initial state
948  // changed to public function (HJK)
949  void reset_all_buffers();
950 
951  static const unsigned int INITIAL_BUFF_SIZE;
952 
953  //! set the verbosity level of output from this pcomm
954  void set_debug_verbosity( int verb );
955 
956  //! get the verbosity level of output from this pcomm
957  int get_debug_verbosity();
958 
959  /* \brief Gather tag value from entities down to a specified root proc
960  * This function gathers data from a domain-decomposed mesh onto a global mesh
961  * represented on the root processor. On the root, this gather mesh is distinct from
962  * the root's domain-decomposed subdomain. Entities are matched by global id, or by
963  * another tag if its handle is input. The dimension of all entities in gather_ents should
964  * be the same, since this is the dimension of entities in gather_set that are queried for
965  * matching global id tags.
966  * \param gather_ents (Local) entities from which to gather data
967  * \param tag_handle Tag whose values are being gathered
968  * \param id_tag Tag to use for matching entities (global id used by default)
969  * \param gather_set On root, set containing global mesh onto which to put data
970  * \param root_proc_rank Rank of the specified root processor (default rank is 0)
971  */
972  ErrorCode gather_data( Range& gather_ents,
973  Tag& tag_handle,
974  Tag id_tag = 0,
975  EntityHandle gather_set = 0,
976  int root_proc_rank = 0 );
977 
978  /* \brief communicate extra points positions on boundary
979  * This function is called after intersection of 2 meshes, to settle the
980  * position of the intersection points on the boundary (interface)
981  * The initial mesh distributed on each processor is decomposed after
982  * intersection with another mesh, such as that new points are created on the
983  * boundary. these points should better match at the interface !
984  * we perform an extra caution step, to ensure the robustness of the
985  * intersection algorithm; only shared edges extra nodes
986  * will be actually needed to be communicated, but we just pass by reference
987  * the whole extraNodesVec structure, we do
988  * not need to construct another data structure
989  * The node positions on edges that are owned will be communicated to other
990  * processors
991  *
992  * \param edges total range of entities
993  * \param shared_edges_owned edges for which to communicate data
994  * \param extraNodesVec handles of intersection vertices on all edges;
995  */
996  ErrorCode settle_intersection_points( Range& edges,
997  Range& shared_edges_owned,
998  std::vector< std::vector< EntityHandle >* >& extraNodesVec,
999  double tolerance );
1000 
1001  /* \brief delete entities from moab database
1002  * will check the shared ents array, and clean it if necessary
1003  *
1004  */
1005  ErrorCode delete_entities( Range& to_delete );
1006 
1007  /*
1008  * \brief correct multi-sharing info for thin layers
1009  *
1010  * will be used for at least 3 processes, when there are thin ghost layers
1011  * right now it is public, for allowing users to call it directly
1012  * eventually, it should become private, and be called automatically
1013  */
1014 
1015  ErrorCode correct_thin_ghost_layers();
1016 
1017  private:
1018  ErrorCode reduce_void( int tag_data_type, const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals );
1019 
1020  template < class T >
1021  ErrorCode reduce( const MPI_Op mpi_op, int num_ents, void* old_vals, void* new_vals );
1022 
1023  void print_debug_isend( int from, int to, unsigned char* buff, int tag, int size );
1024 
1025  void print_debug_irecv( int to, int from, unsigned char* buff, int size, int tag, int incoming );
1026 
1027  void print_debug_recd( MPI_Status status );
1028 
1029  void print_debug_waitany( std::vector< MPI_Request >& reqs, int tag, int proc );
1030 
1031  // common initialization code, called from various constructors
1032  void initialize();
1033 
1034  ErrorCode set_sharing_data( EntityHandle ent,
1035  unsigned char pstatus,
1036  int old_nump,
1037  int new_nump,
1038  int* ps,
1039  EntityHandle* hs );
1040 
1041  ErrorCode check_clean_iface( Range& allsent );
1042 
1043  void define_mpe();
1044 
1045  ErrorCode get_sent_ents( const bool is_iface,
1046  const int bridge_dim,
1047  const int ghost_dim,
1048  const int num_layers,
1049  const int addl_ents,
1050  Range* sent_ents,
1051  Range& allsent,
1052  TupleList& entprocs );
1053 
1054  /** \brief Set pstatus values on entities
1055  *
1056  * \param pstatus_ents Entities to be set
1057  * \param pstatus_val Pstatus value to be set
1058  * \param lower_dim_ents If true, lower-dimensional ents (incl. vertices) set too
1059  * (and created if they don't exist)
1060  * \param verts_too If true, vertices also set
1061  * \param operation If UNION, pstatus_val is OR-d with existing value, otherwise
1062  * existing value is over-written
1063  */
1064  ErrorCode set_pstatus_entities( Range& pstatus_ents,
1065  unsigned char pstatus_val,
1066  bool lower_dim_ents = false,
1067  bool verts_too = true,
1068  int operation = Interface::UNION );
1069 
1070  /** \brief Set pstatus values on entities (vector-based function)
1071  *
1072  * \param pstatus_ents Entities to be set
1073  * \param pstatus_val Pstatus value to be set
1074  * \param lower_dim_ents If true, lower-dimensional ents (incl. vertices) set too
1075  * (and created if they don't exist)
1076  * \param verts_too If true, vertices also set
1077  * \param operation If UNION, pstatus_val is OR-d with existing value, otherwise
1078  * existing value is over-written
1079  */
1080  ErrorCode set_pstatus_entities( EntityHandle* pstatus_ents,
1081  int num_ents,
1082  unsigned char pstatus_val,
1083  bool lower_dim_ents = false,
1084  bool verts_too = true,
1085  int operation = Interface::UNION );
1086 
1087  //! estimate size required to pack entities
1088  int estimate_ents_buffer_size( Range& entities, const bool store_remote_handles );
1089 
1090  //! estimate size required to pack sets
1091  int estimate_sets_buffer_size( Range& entities, const bool store_remote_handles );
1092 
1093  //! send the indicated buffer, possibly sending size first
1094  ErrorCode send_buffer( const unsigned int to_proc,
1095  Buffer* send_buff,
1096  const int msg_tag,
1097  MPI_Request& send_req,
1098  MPI_Request& ack_recv_req,
1099  int* ack_buff,
1100  int& this_incoming,
1101  int next_mesg_tag = -1,
1102  Buffer* next_recv_buff = NULL,
1103  MPI_Request* next_recv_req = NULL,
1104  int* next_incoming = NULL );
1105 
1106  //! process incoming message; if longer than the initial size, post
1107  //! recv for next part then send ack; if ack, send second part; else
1108  //! indicate that we're done and buffer is ready for processing
1109  ErrorCode recv_buffer( int mesg_tag_expected,
1110  const MPI_Status& mpi_status,
1111  Buffer* recv_buff,
1112  MPI_Request& recv_2nd_req,
1113  MPI_Request& ack_req,
1114  int& this_incoming,
1115  Buffer* send_buff,
1116  MPI_Request& send_req,
1117  MPI_Request& sent_ack_req,
1118  bool& done,
1119  Buffer* next_buff = NULL,
1120  int next_tag = -1,
1121  MPI_Request* next_req = NULL,
1122  int* next_incoming = NULL );
1123 
1124  //! pack a range of entities with equal # verts per entity, along with
1125  //! the range on the sending proc
1126  ErrorCode pack_entity_seq( const int nodes_per_entity,
1127  const bool store_remote_handles,
1128  const int to_proc,
1129  Range& these_ents,
1130  std::vector< EntityHandle >& entities,
1131  Buffer* buff );
1132 
1133  ErrorCode print_buffer( unsigned char* buff_ptr, int mesg_type, int from_proc, bool sent );
1134 
1135  //! for all the entities in the received buffer; for each, save
1136  //! entities in this instance which match connectivity, or zero if none found
1137  ErrorCode unpack_iface_entities( unsigned char*& buff_ptr,
1138  const int from_proc,
1139  const int ind,
1140  std::vector< EntityHandle >& recd_ents );
1141 
1142  ErrorCode pack_sets( Range& entities, Buffer* buff, const bool store_handles, const int to_proc );
1143 
1144  ErrorCode unpack_sets( unsigned char*& buff_ptr,
1145  std::vector< EntityHandle >& entities,
1146  const bool store_handles,
1147  const int to_proc );
1148 
1149  ErrorCode pack_adjacencies( Range& entities,
1150  Range::const_iterator& start_rit,
1151  Range& whole_range,
1152  unsigned char*& buff_ptr,
1153  int& count,
1154  const bool just_count,
1155  const bool store_handles,
1156  const int to_proc );
1157 
1158  ErrorCode unpack_adjacencies( unsigned char*& buff_ptr,
1159  Range& entities,
1160  const bool store_handles,
1161  const int from_proc );
1162 
1163  /* \brief Unpack message with remote handles (const pointer to buffer)
1164  */
1165  ErrorCode unpack_remote_handles( unsigned int from_proc,
1166  const unsigned char* buff_ptr,
1167  std::vector< EntityHandle >& L2hloc,
1168  std::vector< EntityHandle >& L2hrem,
1169  std::vector< unsigned int >& L2p );
1170 
1171  //! given connectivity and type, find an existing entity, if there is one
1172  ErrorCode find_existing_entity( const bool is_iface,
1173  const int owner_p,
1174  const EntityHandle owner_h,
1175  const int num_ents,
1176  const EntityHandle* connect,
1177  const int num_connect,
1178  const EntityType this_type,
1179  std::vector< EntityHandle >& L2hloc,
1180  std::vector< EntityHandle >& L2hrem,
1181  std::vector< unsigned int >& L2p,
1182  EntityHandle& new_h );
1183 
1184  ErrorCode build_sharedhps_list( const EntityHandle entity,
1185  const unsigned char pstatus,
1186  const int sharedp,
1187  const std::set< unsigned int >& procs,
1188  unsigned int& num_ents,
1189  int* tmp_procs,
1190  EntityHandle* tmp_handles );
1191 
1192  /**\brief Get list of tags for which to exchange data
1193  *
1194  * Get tags and entities for which to exchange tag data. This function
1195  * was originally part of 'pack_tags' requested with the
1196  * 'all_possible_tags' parameter.
1197  *
1198  *\param all_entities Input. The set of entities for which data is to
1199  * be communicated.
1200  *\param all_tags Output. Populated with the handles of tags to be
1201  * sent.
1202  *\param tag_ranges Output. For each corresponding tag in all_tags, the
1203  * subset of 'all_entities' for which a tag value has
1204  * been set.
1205  */
1206  ErrorCode get_tag_send_list( const Range& all_entities,
1207  std::vector< Tag >& all_tags,
1208  std::vector< Range >& tag_ranges );
1209 
1210  /**\brief Serialize entity tag data
1211  *
1212  * This function operates in two passes. The first phase,
1213  * specified by 'just_count == true' calculates the necessary
1214  * buffer size for the serialized data. The second phase
1215  * writes the actual binary serialized representation of the
1216  * data to the passed buffer.
1217  *
1218  *\NOTE First two arguments are not used. (Legacy interface?)
1219  *
1220  *\param entities NOT USED
1221  *\param start_rit NOT USED
1222  *\param whole_range Should be the union of the sets of entities for
1223  * which tag values are to be serialized. Also
1224  * specifies ordering for indexes for tag values and
1225  * serves as the superset from which to compose entity
1226  * lists from individual tags if just_count and
1227  * all_possible_tags are both true.
1228  *\param buff_ptr Buffer into which to write binary serialized data
1229  *\param count Output: The size of the serialized data is added
1230  * to this parameter. NOTE: Should probably initialize
1231  * to zero before calling.
1232  *\param just_count If true, just calculate the buffer size required to
1233  * hold the serialized data. Will also append to
1234  * 'all_tags' and 'tag_ranges' if all_possible_tags
1235  * == true.
1236  *\param store_handles The data for each tag is preceded by a list of
1237  * EntityHandles designating the entity each of
1238  * the subsequent tag values corresponds to. This value
1239  * may be one of:
1240  * 1) If store_handles == false:
1241  * An invalid handle composed of {MBMAXTYPE,idx}, where
1242  * idx is the position of the entity in "whole_range".
1243  * 2) If store_hanldes == true and a valid remote
1244  * handle exists, the remote handle.
1245  * 3) If store_hanldes == true and no valid remote
1246  * handle is defined for the entity, the same as 1).
1247  *\param to_proc If 'store_handles' is true, the processor rank for
1248  * which to store the corresponding remote entity
1249  * handles.
1250  *\param all_tags List of tags to write
1251  *\param tag_ranges List of entities to serialize tag data, one
1252  * for each corresponding tag handle in 'all_tags.
1253  */
1254  ErrorCode pack_tags( Range& entities,
1255  const std::vector< Tag >& src_tags,
1256  const std::vector< Tag >& dst_tags,
1257  const std::vector< Range >& tag_ranges,
1258  Buffer* buff,
1259  const bool store_handles,
1260  const int to_proc );
1261 
1262  /**\brief Calculate buffer size required to pack tag data
1263  *\param source_tag The tag for which data will be serialized
1264  *\param entities The entities for which tag values will be serialized
1265  *\param count_out Output: The required buffer size, in bytes.
1266  */
1267  ErrorCode packed_tag_size( Tag source_tag, const Range& entities, int& count_out );
1268 
1269  /**\brief Serialize tag data
1270  *\param source_tag The tag for which data will be serialized
1271  *\param destination_tag Tag in which to store unpacked tag data. Typically
1272  * the same as source_tag.
1273  *\param entities The entities for which tag values will be serialized
1274  *\param whole_range Calculate entity indices as location in this range
1275  *\param buff_ptr Input/Output: As input, pointer to the start of the
1276  * buffer in which to serialize data. As output, the
1277  * position just passed the serialized data.
1278  *\param count_out Output: The required buffer size, in bytes.
1279  *\param store_handles The data for each tag is preceded by a list of
1280  * EntityHandles designating the entity each of
1281  * the subsequent tag values corresponds to. This value
1282  * may be one of:
1283  * 1) If store_handles == false:
1284  * An invalid handle composed of {MBMAXTYPE,idx}, where
1285  * idx is the position of the entity in "whole_range".
1286  * 2) If store_hanldes == true and a valid remote
1287  * handle exists, the remote handle.
1288  * 3) If store_hanldes == true and no valid remote
1289  * handle is defined for the entity, the same as 1).
1290  *\param to_proc If 'store_handles' is true, the processor rank for
1291  * which to store the corresponding remote entity
1292  * handles.
1293  */
1294  ErrorCode pack_tag( Tag source_tag,
1295  Tag destination_tag,
1296  const Range& entities,
1297  const std::vector< EntityHandle >& whole_range,
1298  Buffer* buff,
1299  const bool store_remote_handles,
1300  const int to_proc );
1301 
1302  ErrorCode unpack_tags( unsigned char*& buff_ptr,
1303  std::vector< EntityHandle >& entities,
1304  const bool store_handles,
1305  const int to_proc,
1306  const MPI_Op* const mpi_op = NULL );
1307 
1308  ErrorCode tag_shared_verts( TupleList& shared_verts,
1309  Range* skin_ents,
1310  std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs,
1311  Range& proc_verts );
1312 
1313  ErrorCode get_proc_nvecs( int resolve_dim,
1314  int shared_dim,
1315  Range* skin_ents,
1316  std::map< std::vector< int >, std::vector< EntityHandle > >& proc_nvecs );
1317 
1318  // after verifying shared entities, now parent/child links between sets can be established
1319  ErrorCode create_iface_pc_links();
1320 
1321  //! pack a range map with keys in this_range and values a contiguous series
1322  //! of handles starting at actual_start
1323  ErrorCode pack_range_map( Range& this_range, EntityHandle actual_start, HandleMap& handle_map );
1324 
1325  //! returns true if the set is an interface shared with to_proc
1326  bool is_iface_proc( EntityHandle this_set, int to_proc );
1327 
1328  //! for any remote_handles set to zero, remove corresponding sent_ents from
1329  //! iface_sets corresponding to from_proc
1330  ErrorCode update_iface_sets( Range& sent_ents, std::vector< EntityHandle >& remote_handles, int from_proc );
1331 
1332  //! for specified bridge/ghost dimension, to_proc, and number
1333  //! of layers, get the entities to be ghosted, and info on additional procs
1334  //! needing to communicate with to_proc
1335  ErrorCode get_ghosted_entities( int bridge_dim,
1336  int ghost_dim,
1337  int to_proc,
1338  int num_layers,
1339  int addl_ents,
1340  Range& ghosted_ents );
1341 
1342  //! add vertices adjacent to entities in this list
1343  ErrorCode add_verts( Range& sent_ents );
1344 
1345  //! Every processor sends shared entity handle data to every other processor
1346  //! that it shares entities with. Passed back map is all received data,
1347  //! indexed by processor ID. This function is intended to be used for
1348  //! debugging.
1349  ErrorCode exchange_all_shared_handles( std::vector< std::vector< SharedEntityData > >& send_data,
1350  std::vector< std::vector< SharedEntityData > >& result );
1351 
1352  //! replace handles in from_vec with corresponding handles on
1353  //! to_proc (by checking shared[p/h]_tag and shared[p/h]s_tag;
1354  //! if no remote handle and new_ents is non-null, substitute
1355  //! instead CREATE_HANDLE(MBMAXTYPE, index) where index is handle's
1356  //! position in new_ents
1357  ErrorCode get_remote_handles( const bool store_remote_handles,
1358  EntityHandle* from_vec,
1359  EntityHandle* to_vec_tmp,
1360  int num_ents,
1361  int to_proc,
1362  const std::vector< EntityHandle >& new_ents );
1363 
1364  //! same as other version, except from_range and to_range should be
1365  //! different here
1366  ErrorCode get_remote_handles( const bool store_remote_handles,
1367  const Range& from_range,
1368  Range& to_range,
1369  int to_proc,
1370  const std::vector< EntityHandle >& new_ents );
1371 
1372  //! same as other version, except packs range into vector
1373  ErrorCode get_remote_handles( const bool store_remote_handles,
1374  const Range& from_range,
1375  EntityHandle* to_vec,
1376  int to_proc,
1377  const std::vector< EntityHandle >& new_ents );
1378 
1379  //! goes through from_vec, and for any with type MBMAXTYPE, replaces with
1380  //! new_ents value at index corresponding to id of entity in from_vec
1381  ErrorCode get_local_handles( EntityHandle* from_vec, int num_ents, const Range& new_ents );
1382 
1383  //! same as above except puts results in range
1384  ErrorCode get_local_handles( const Range& remote_handles,
1385  Range& local_handles,
1386  const std::vector< EntityHandle >& new_ents );
1387 
1388  //! same as above except gets new_ents from vector
1389  ErrorCode get_local_handles( EntityHandle* from_vec, int num_ents, const std::vector< EntityHandle >& new_ents );
1390 
1391  ErrorCode update_remote_data( Range& local_range,
1392  Range& remote_range,
1393  int other_proc,
1394  const unsigned char add_pstat );
1395 
1396  ErrorCode update_remote_data( const EntityHandle new_h,
1397  const int* ps,
1398  const EntityHandle* hs,
1399  const int num_ps,
1400  const unsigned char add_pstat );
1401 
1402  ErrorCode update_remote_data_old( const EntityHandle new_h,
1403  const int* ps,
1404  const EntityHandle* hs,
1405  const int num_ps,
1406  const unsigned char add_pstat );
1407 
1408  /** \brief Set pstatus tag interface bit on entities in sets passed in
1409  */
1410  ErrorCode tag_iface_entities();
1411 
1412  //! add a pc to the iface instance tag PARALLEL_COMM
1413  int add_pcomm( ParallelComm* pc );
1414 
1415  //! remove a pc from the iface instance tag PARALLEL_COMM
1416  void remove_pcomm( ParallelComm* pc );
1417 
1418  //! check entities to make sure there are no zero-valued remote handles
1419  //! where they shouldn't be
1420  ErrorCode check_sent_ents( Range& allsent );
1421 
1422  //! assign entities to the input processor part
1423  ErrorCode assign_entities_part( std::vector< EntityHandle >& entities, const int proc );
1424 
1425  //! remove entities to the input processor part
1426  ErrorCode remove_entities_part( Range& entities, const int proc );
1427 
1428  //! MB interface associated with this writer
1429  Interface* mbImpl;
1430 
1431  //! Proc config object, keeps info on parallel stuff
1432  ProcConfig procConfig;
1433 
1434  //! Sequence manager, to get more efficient access to entities
1435  SequenceManager* sequenceManager;
1436 
1437  //! Error handler
1438  Error* errorHandler;
1439 
1440  //! more data buffers, proc-specific
1441  std::vector< Buffer* > localOwnedBuffs, remoteOwnedBuffs;
1442 
1443  //! reset message buffers to their initial state
1444  // void reset_all_buffers();
1445 
1446  //! delete all buffers, freeing up any memory held by them
1447  void delete_all_buffers();
1448 
1449  //! request objects, may be used if store_remote_handles is used
1450  std::vector< MPI_Request > sendReqs;
1451 
1452  //! receive request objects
1453  std::vector< MPI_Request > recvReqs, recvRemotehReqs;
1454 
1455  //! processor rank for each buffer index
1456  std::vector< unsigned int > buffProcs;
1457 
1458  //! the partition, interface sets for this comm'n instance
1459  Range partitionSets, interfaceSets;
1460 
1461  //! all local entities shared with others, whether ghost or ghosted
1462  std::set< EntityHandle > sharedEnts;
1463 
1464  //! tags used to save sharing procs and handles
1465  Tag sharedpTag, sharedpsTag, sharedhTag, sharedhsTag, pstatusTag, ifaceSetsTag, partitionTag;
1466 
1467  int globalPartCount; //!< Cache of global part count
1468 
1469  EntityHandle partitioningSet; //!< entity set containing all parts
1470 
1471  std::ofstream myFile;
1472 
1473  int pcommID;
1474 
1475  int ackbuff;
1476 
1477  //! used to set verbosity level and to report output
1478  DebugOutput* myDebug;
1479 
1480  //! Data about shared sets
1481  SharedSetData* sharedSetData;
1482 };
1483 
1484 inline ParallelComm::Buffer::Buffer( const Buffer& other_buff )
1485 {
1486  alloc_size = other_buff.alloc_size;
1487  mem_ptr = (unsigned char*)malloc( alloc_size );
1488  memcpy( mem_ptr, other_buff.mem_ptr, alloc_size );
1489  buff_ptr = mem_ptr + ( other_buff.buff_ptr - other_buff.mem_ptr );
1490 }
1491 
1492 inline ParallelComm::Buffer::Buffer( unsigned int new_size ) : mem_ptr( NULL ), buff_ptr( NULL ), alloc_size( 0 )
1493 {
1494  if( new_size ) this->reserve( new_size );
1495 }
1496 
1497 inline ParallelComm::Buffer::~Buffer()
1498 {
1499  if( mem_ptr )
1500  {
1501  free( mem_ptr );
1502  mem_ptr = NULL;
1503  }
1504 }
1505 
1506 #define DEBUG_BUFFER 0
1507 
1508 inline void ParallelComm::Buffer::reserve( unsigned int new_size )
1509 {
1510 
1511 #ifdef DEBUG_BUFFER
1512  int tmp_pos = 0;
1513  if( mem_ptr )
1514  {
1515  tmp_pos = buff_ptr - mem_ptr;
1516  }
1517  buff_ptr = (unsigned char*)malloc( new_size );
1518  assert( 0 <= tmp_pos && tmp_pos <= (int)alloc_size );
1519  if( tmp_pos ) memcpy( buff_ptr, mem_ptr, tmp_pos );
1520  if( mem_ptr ) free( mem_ptr );
1521  mem_ptr = buff_ptr;
1522  alloc_size = new_size;
1523  buff_ptr = mem_ptr + tmp_pos;
1524 #else
1525  if( mem_ptr && alloc_size < new_size )
1526  {
1527  size_t tmp_pos = mem_ptr ? buff_ptr - mem_ptr : 0;
1528  mem_ptr = (unsigned char*)realloc( mem_ptr, new_size );
1529  alloc_size = new_size;
1530  buff_ptr = mem_ptr + tmp_pos;
1531  }
1532  else if( !mem_ptr )
1533  {
1534  mem_ptr = (unsigned char*)malloc( new_size );
1535  alloc_size = new_size;
1536  buff_ptr = mem_ptr;
1537  }
1538 #endif
1539 }
1540 
1541 inline void ParallelComm::Buffer::check_space( unsigned int addl_space )
1542 {
1543  assert( buff_ptr >= mem_ptr && buff_ptr <= mem_ptr + alloc_size );
1544  unsigned int new_size = buff_ptr - mem_ptr + addl_space;
1545  if( new_size > alloc_size ) reserve( 3 * new_size / 2 );
1546 }
1547 
1548 inline void ParallelComm::reset_all_buffers()
1549 {
1550  std::vector< Buffer* >::iterator vit;
1551  for( vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); ++vit )
1552  ( *vit )->reset_buffer();
1553  for( vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); ++vit )
1554  ( *vit )->reset_buffer();
1555 }
1556 
1557 inline void ParallelComm::delete_all_buffers()
1558 {
1559  std::vector< Buffer* >::iterator vit;
1560  for( vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); ++vit )
1561  delete( *vit );
1562  localOwnedBuffs.clear();
1563 
1564  for( vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); ++vit )
1565  delete( *vit );
1566  remoteOwnedBuffs.clear();
1567 }
1568 
1569 inline const std::vector< unsigned int >& ParallelComm::buff_procs() const
1570 {
1571  return buffProcs;
1572 }
1573 
1574 inline ErrorCode ParallelComm::get_shared_proc_tags( Tag& sharedp,
1575  Tag& sharedps,
1576  Tag& sharedh,
1577  Tag& sharedhs,
1578  Tag& pstatus )
1579 {
1580  sharedp = sharedp_tag();
1581  sharedps = sharedps_tag();
1582  sharedh = sharedh_tag();
1583  sharedhs = sharedhs_tag();
1584  pstatus = pstatus_tag();
1585 
1586  return MB_SUCCESS;
1587 }
1588 
1589 inline ErrorCode ParallelComm::exchange_tags( const char* tag_name, const Range& entities )
1590 {
1591  // get the tag handle
1592  std::vector< Tag > tags( 1 );
1593  ErrorCode result = mbImpl->tag_get_handle( tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY );
1594  if( MB_SUCCESS != result )
1595  return result;
1596  else if( !tags[0] )
1597  return MB_TAG_NOT_FOUND;
1598 
1599  return exchange_tags( tags, tags, entities );
1600 }
1601 
1602 inline ErrorCode ParallelComm::exchange_tags( Tag tagh, const Range& entities )
1603 {
1604  // get the tag handle
1605  std::vector< Tag > tags;
1606  tags.push_back( tagh );
1607 
1608  return exchange_tags( tags, tags, entities );
1609 }
1610 
1611 inline ErrorCode ParallelComm::reduce_tags( const char* tag_name, const MPI_Op mpi_op, const Range& entities )
1612 {
1613  // get the tag handle
1614  std::vector< Tag > tags( 1 );
1615  ErrorCode result = mbImpl->tag_get_handle( tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY );
1616  if( MB_SUCCESS != result )
1617  return result;
1618  else if( !tags[0] )
1619  return MB_TAG_NOT_FOUND;
1620 
1621  return reduce_tags( tags, tags, mpi_op, entities );
1622 }
1623 
1624 inline ErrorCode ParallelComm::reduce_tags( Tag tagh, const MPI_Op mpi_op, const Range& entities )
1625 {
1626  // get the tag handle
1627  std::vector< Tag > tags;
1628  tags.push_back( tagh );
1629 
1630  return reduce_tags( tags, tags, mpi_op, entities );
1631 }
1632 
1633 inline ErrorCode ParallelComm::get_comm_procs( std::set< unsigned int >& procs )
1634 {
1635  ErrorCode result = get_interface_procs( procs );
1636  if( MB_SUCCESS != result ) return result;
1637 
1638  std::copy( buffProcs.begin(), buffProcs.end(), std::inserter( procs, procs.begin() ) );
1639 
1640  return MB_SUCCESS;
1641 }
1642 
1643 inline ErrorCode ParallelComm::get_owner( EntityHandle entity, int& owner )
1644 {
1645  EntityHandle tmp_handle;
1646  return get_owner_handle( entity, owner, tmp_handle );
1647 }
1648 
1649 /* \brief Unpack message with remote handles (const pointer to buffer)
1650  */
1651 inline ErrorCode ParallelComm::unpack_remote_handles( unsigned int from_proc,
1652  const unsigned char* buff_ptr,
1653  std::vector< EntityHandle >& L2hloc,
1654  std::vector< EntityHandle >& L2hrem,
1655  std::vector< unsigned int >& L2p )
1656 {
1657  // cast away const-ness, we won't be passing back a modified ptr
1658  unsigned char* tmp_buff = const_cast< unsigned char* >( buff_ptr );
1659  return unpack_remote_handles( from_proc, tmp_buff, L2hloc, L2hrem, L2p );
1660 }
1661 
1662 inline void ParallelComm::set_rank( unsigned int r )
1663 {
1664  procConfig.proc_rank( r );
1665  if( procConfig.proc_size() < r ) procConfig.proc_size( r + 1 );
1666 }
1667 
1668 inline void ParallelComm::set_size( unsigned int s )
1669 {
1670  procConfig.proc_size( s );
1671 }
1672 
1673 inline ErrorCode ParallelComm::get_sharing_data( const EntityHandle* entities,
1674  int num_entities,
1675  std::set< int >& procs,
1676  int op )
1677 {
1678  Range dum_range;
1679  // cast away constness 'cuz the range is passed as const
1680  EntityHandle* ents_cast = const_cast< EntityHandle* >( entities );
1681  std::copy( ents_cast, ents_cast + num_entities, range_inserter( dum_range ) );
1682  return get_sharing_data( dum_range, procs, op );
1683 }
1684 
1685 inline ErrorCode ParallelComm::get_sharing_data( const EntityHandle entity,
1686  int* ps,
1687  EntityHandle* hs,
1688  unsigned char& pstat,
1689  int& num_ps )
1690 {
1691  unsigned int dum_ps;
1692  ErrorCode result = get_sharing_data( entity, ps, hs, pstat, dum_ps );
1693  if( MB_SUCCESS == result ) num_ps = dum_ps;
1694  return result;
1695 }
1696 
1697 inline void ParallelComm::set_send_request( int n_request )
1698 {
1699  sendReqs.resize( n_request, MPI_REQUEST_NULL );
1700 }
1701 
1702 inline void ParallelComm::set_recv_request( int n_request )
1703 {
1704  recvReqs.resize( n_request, MPI_REQUEST_NULL );
1705 }
1706 } // namespace moab
1707 
1708 #endif