2: #include <petscsys.h>        /*I  "petscsys.h"  I*/
  3: /*
  4:     Note that tag of 0 is ok because comm is a private communicator
  5:   generated below just for these routines.
  6: */

 10: PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng)
 11: {
 13:   PetscMPIInt    rank,size,tag = 0;
 14:   MPI_Status     status;

 17:   MPI_Comm_size(comm,&size);
 18:   if (size == 1) return(0);
 19:   MPI_Comm_rank(comm,&rank);
 20:   if (rank) {
 21:     MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);
 22:   }
 23:   /* Send to the next process in the group unless we are the last process */
 24:   if ((rank % ng) < ng - 1 && rank != size - 1) {
 25:     MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);
 26:   }
 27:   return(0);
 28: }

 32: PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng)
 33: {
 35:   PetscMPIInt    rank,size,tag = 0;
 36:   MPI_Status     status;

 39:   MPI_Comm_rank(comm,&rank);
 40:   MPI_Comm_size(comm,&size);
 41:   if (size == 1) return(0);

 43:   /* Send to the first process in the next group */
 44:   if ((rank % ng) == ng - 1 || rank == size - 1) {
 45:     MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);
 46:   }
 47:   if (!rank) {
 48:     MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);
 49:   }
 50:   return(0);
 51: }

 53: /* ---------------------------------------------------------------------*/
 54: /*
 55:     The variable Petsc_Seq_keyval is used to indicate an MPI attribute that
 56:   is attached to a communicator that manages the sequential phase code below.
 57: */
 58: static int Petsc_Seq_keyval = MPI_KEYVAL_INVALID;

 62: /*@
 63:    PetscSequentialPhaseBegin - Begins a sequential section of code.  

 65:    Collective on MPI_Comm

 67:    Input Parameters:
 68: +  comm - Communicator to sequentialize.  
 69: -  ng   - Number in processor group.  This many processes are allowed to execute
 70:    at the same time (usually 1)

 72:    Level: intermediate

 74:    Notes:
 75:    PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a
 76:    way to force a section of code to be executed by the processes in
 77:    rank order.  Typically, this is done with
 78: .vb
 79:       PetscSequentialPhaseBegin(comm, 1);
 80:       <code to be executed sequentially>
 81:       PetscSequentialPhaseEnd(comm, 1);
 82: .ve

 84:    Often, the sequential code contains output statements (e.g., printf) to
 85:    be executed.  Note that you may need to flush the I/O buffers before
 86:    calling PetscSequentialPhaseEnd().  Also, note that some systems do
 87:    not propagate I/O in any order to the controling terminal (in other words, 
 88:    even if you flush the output, you may not get the data in the order
 89:    that you want).

 91: .seealso: PetscSequentialPhaseEnd()

 93:    Concepts: sequential stage

 95: @*/
 96: PetscErrorCode  PetscSequentialPhaseBegin(MPI_Comm comm,int ng)
 97: {
 99:   PetscMPIInt    size;
100:   MPI_Comm       local_comm,*addr_local_comm;

103:   MPI_Comm_size(comm,&size);
104:   if (size == 1) return(0);

106:   /* Get the private communicator for the sequential operations */
107:   if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) {
108:     MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);
109:   }

111:   MPI_Comm_dup(comm,&local_comm);
112:   PetscMalloc(sizeof(MPI_Comm),&addr_local_comm);
113:   *addr_local_comm = local_comm;
114:   MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);
115:   PetscSequentialPhaseBegin_Private(local_comm,ng);
116:   return(0);
117: }

121: /*@
122:    PetscSequentialPhaseEnd - Ends a sequential section of code.

124:    Collective on MPI_Comm

126:    Input Parameters:
127: +  comm - Communicator to sequentialize.  
128: -  ng   - Number in processor group.  This many processes are allowed to execute
129:    at the same time (usually 1)

131:    Level: intermediate

133:    Notes:
134:    See PetscSequentialPhaseBegin() for more details.

136: .seealso: PetscSequentialPhaseBegin()

138:    Concepts: sequential stage

140: @*/
141: PetscErrorCode  PetscSequentialPhaseEnd(MPI_Comm comm,int ng)
142: {
144:   PetscMPIInt    size,flag;
145:   MPI_Comm       local_comm,*addr_local_comm;

148:   MPI_Comm_size(comm,&size);
149:   if (size == 1) return(0);

151:   MPI_Attr_get(comm,Petsc_Seq_keyval,(void **)&addr_local_comm,&flag);
152:   if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()");
153:   local_comm = *addr_local_comm;

155:   PetscSequentialPhaseEnd_Private(local_comm,ng);

157:   PetscFree(addr_local_comm);
158:   MPI_Comm_free(&local_comm);
159:   MPI_Attr_delete(comm,Petsc_Seq_keyval);
160:   return(0);
161: }