Actual source code: vscreate.c

petsc-3.13.6 2020-09-29
Report Typos and Errors
  1: /*
  2:     The VECSCATTER (vec scatter) interface routines, callable by users.
  3: */

  5:  #include <petsc/private/vecscatterimpl.h>

  7: /* Logging support */
  8: PetscClassId VEC_SCATTER_CLASSID;

 10: PetscFunctionList VecScatterList              = NULL;
 11: PetscBool         VecScatterRegisterAllCalled = PETSC_FALSE;

 13: /*@C
 14:   VecScatterSetType - Builds a vector scatter, for a particular vector scatter implementation.

 16:   Collective on VecScatter

 18:   Input Parameters:
 19: + vscat - The vector scatter object
 20: - type - The name of the vector scatter type

 22:   Options Database Key:
 23: . -vecscatter_type <type> - Sets the vector scatter type; use -help for a list
 24:                      of available types

 26:   Notes:
 27:   See "petsc/include/petscvec.h" for available vector scatter types (for instance, VECSCATTERMPI1, or VECSCATTERMPI3NODE).

 29:   Use VecScatterDuplicate() to form additional vectors scatter of the same type as an existing vector scatter.

 31:   Level: intermediate

 33: .seealso: VecScatterGetType(), VecScatterCreate()
 34: @*/
 35: PetscErrorCode VecScatterSetType(VecScatter vscat, VecScatterType type)
 36: {
 37:   PetscBool      match;
 39:   PetscErrorCode (*r)(VecScatter);

 43:   PetscObjectTypeCompare((PetscObject)vscat, type, &match);
 44:   if (match) return(0);

 46:   PetscFunctionListFind(VecScatterList,type,&r);
 47:   if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE, "Unknown vector scatter type: %s",type);

 49:   if (vscat->ops->destroy) {
 50:     (*vscat->ops->destroy)(vscat);
 51:     vscat->ops->destroy = NULL;
 52:   }

 54:   (*r)(vscat);
 55:   return(0);
 56: }

 58: /*@C
 59:   VecScatterGetType - Gets the vector scatter type name (as a string) from the VecScatter.

 61:   Not Collective

 63:   Input Parameter:
 64: . vscat  - The vector scatter

 66:   Output Parameter:
 67: . type - The vector scatter type name

 69:   Level: intermediate

 71: .seealso: VecScatterSetType(), VecScatterCreate()
 72: @*/
 73: PetscErrorCode VecScatterGetType(VecScatter vscat, VecScatterType *type)
 74: {

 80:   VecScatterRegisterAll();
 81:   *type = ((PetscObject)vscat)->type_name;
 82:   return(0);
 83: }

 85: /*@
 86:   VecScatterSetFromOptions - Configures the vector scatter from the options database.

 88:   Collective on VecScatter

 90:   Input Parameter:
 91: . vscat - The vector scatter

 93:   Notes:
 94:     To see all options, run your program with the -help option, or consult the users manual.
 95:           Must be called before VecScatterSetUp() but before the vector scatter is used.

 97:   Level: beginner


100: .seealso: VecScatterCreate(), VecScatterDestroy(), VecScatterSetUp()
101: @*/
102: PetscErrorCode VecScatterSetFromOptions(VecScatter vscat)
103: {
105:   PetscBool      opt;
106:   VecScatterType defaultType;
107:   char           typeName[256];
108:   PetscMPIInt    size;


113:   PetscObjectOptionsBegin((PetscObject)vscat);
114:   MPI_Comm_size(PetscObjectComm((PetscObject)vscat), &size);

116:   /* Handle vector type options */
117:   if (((PetscObject)vscat)->type_name) {
118:     defaultType = ((PetscObject)vscat)->type_name;
119:   } else {
120:     if (size > 1) defaultType = VECSCATTERMPI1;
121:     else defaultType = VECSCATTERSEQ;
122:   }

124:   VecScatterRegisterAll();
125:   PetscOptionsFList("-vecscatter_type","Vector Scatter type","VecScatterSetType",VecScatterList,defaultType,typeName,256,&opt);
126:   if (size > 1 && opt) {
127:     VecScatterSetType(vscat,typeName);
128:   } else {
129:     VecScatterSetType(vscat,defaultType);
130:   }

132:   vscat->beginandendtogether = PETSC_FALSE;
133:   PetscOptionsBool("-vecscatter_merge","Use combined (merged) vector scatter begin and end","VecScatterCreate",vscat->beginandendtogether,&vscat->beginandendtogether,NULL);
134:   if (vscat->beginandendtogether) {
135:     PetscInfo(vscat,"Using combined (merged) vector scatter begin and end\n");
136:   }

138:   vscat->packongpu = PETSC_TRUE;
139:   PetscOptionsBool("-vecscatter_packongpu","For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI","VecScatterCreate",vscat->packongpu,&vscat->packongpu,NULL);
140:   if (vscat->packongpu) {
141:     PetscInfo(vscat,"For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI\n");
142:   }

144:   PetscOptionsEnd();
145:   return(0);
146: }

148: /*@C
149:   VecScatterRegister -  Adds a new vector scatter component implementation

151:   Not Collective

153:   Input Parameters:
154: + name        - The name of a new user-defined creation routine
155: - create_func - The creation routine itself

157:   Notes:
158:   VecScatterRegister() may be called multiple times to add several user-defined vectors

160:   Sample usage:
161: .vb
162:     VecScatterRegister("my_vecscatter",MyVecScatterCreate);
163: .ve

165:   Then, your vector scatter type can be chosen with the procedural interface via
166: .vb
167:     VecScatterCreate(MPI_Comm, VecScatter *);
168:     VecScatterSetType(VecScatter,"my_vectorscatter_name");
169: .ve
170:    or at runtime via the option
171: .vb
172:     -vecscatter_type my_vectorscatter_name
173: .ve

175:   Level: advanced

177: .seealso: VecScatterRegisterAll(), VecScatterRegisterDestroy()
178: @*/
179: PetscErrorCode VecScatterRegister(const char sname[], PetscErrorCode (*function)(VecScatter))
180: {

184:   VecInitializePackage();
185:   PetscFunctionListAdd(&VecScatterList,sname,function);
186:   return(0);
187: }

189: /* ---------------------------------------------------------------- */
190: /*@
191:    VecScatterCreate - Creates a vector scatter context.

193:    Collective on Vec

195:    Input Parameters:
196: +  xin - a vector that defines the shape (parallel data layout of the vector)
197:          of vectors from which we scatter
198: .  yin - a vector that defines the shape (parallel data layout of the vector)
199:          of vectors to which we scatter
200: .  ix - the indices of xin to scatter (if NULL scatters all values)
201: -  iy - the indices of yin to hold results (if NULL fills entire vector yin)

203:    Output Parameter:
204: .  newctx - location to store the new scatter context

206:    Options Database Keys:
207: +  -vecscatter_view         - Prints detail of communications
208: .  -vecscatter_view ::ascii_info    - Print less details about communication
209: .  -vecscatter_merge        - VecScatterBegin() handles all of the communication, VecScatterEnd() is a nop
210:                               eliminates the chance for overlap of computation and communication
211: .  -vecscatter_packtogether - Pack all messages before sending, receive all messages before unpacking
212:                               will make the results of scatters deterministic when otherwise they are not (it may be slower also).
213: .  -vecscatter_type sf      - Use the PetscSF implementation of vecscatter (Default). One can use PetscSF options to control the communication.
214: -  -vecscatter_packongpu    - For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI.
215:                               Otherwise, we might copy a segment encompassing needed entries. Default is TRUE.

217:     Level: intermediate

219:   Notes:
220:    If both xin and yin are parallel, their communicator must be on the same
221:    set of processes, but their process order can be different.
222:    In calls to VecScatter() you can use different vectors than the xin and
223:    yin you used above; BUT they must have the same parallel data layout, for example,
224:    they could be obtained from VecDuplicate().
225:    A VecScatter context CANNOT be used in two or more simultaneous scatters;
226:    that is you cannot call a second VecScatterBegin() with the same scatter
227:    context until the VecScatterEnd() has been called on the first VecScatterBegin().
228:    In this case a separate VecScatter is needed for each concurrent scatter.

230:    Currently the MPI_Send() use PERSISTENT versions.
231:    (this unfortunately requires that the same in and out arrays be used for each use, this
232:     is why  we always need to pack the input into the work array before sending
233:     and unpack upon receiving instead of using MPI datatypes to avoid the packing/unpacking).

235:    Both ix and iy cannot be NULL at the same time.

237:    Use VecScatterCreateToAll() to create a vecscatter that copies an MPI vector to sequential vectors on all MPI ranks.
238:    Use VecScatterCreateToZero() to create a vecscatter that copies an MPI vector to a sequential vector on MPI rank 0.
239:    These special vecscatters have better performance than general ones.

241: .seealso: VecScatterDestroy(), VecScatterCreateToAll(), VecScatterCreateToZero(), PetscSFCreate()
242: @*/
243: PetscErrorCode VecScatterCreate(Vec xin,IS ix,Vec yin,IS iy,VecScatter *newctx)
244: {
245:   VecScatter        ctx;
246:   PetscErrorCode    ierr;
247:   PetscMPIInt       xsize,ysize,result;
248:   MPI_Comm          comm,xcomm,ycomm;

252:   *newctx = NULL;

254:   if (!ix && !iy) SETERRQ(PetscObjectComm((PetscObject)xin),PETSC_ERR_SUP,"Cannot pass default in for both input and output indices");

256:   /* Get comm from xin and yin */
257:   PetscObjectGetComm((PetscObject)xin,&xcomm);
258:   MPI_Comm_size(xcomm,&xsize);
259:   PetscObjectGetComm((PetscObject)yin,&ycomm);
260:   MPI_Comm_size(ycomm,&ysize);
261:   if (xsize > 1 && ysize > 1) {
262:     MPI_Comm_compare(xcomm,ycomm,&result);
263:     if (result == MPI_UNEQUAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"VecScatterCreate: parallel vectors xin and yin must have identical/congruent/similar communicators");
264:   }

266:   /* If xsize==ysize, we use ycomm as comm of ctx, which does affect correctness of -vecscatter_type mpi1. But it does not
267:      affect -vecscatter_type sf, since sf internally uses the roots' comm (equal to xcomm when x is parallel) as the sf
268:      object's comm. One day when we degrade VecScatter from a petsc object to a simple wrapper around PetscSF, we can get
269:      rid of this confusion.
270:    */
271:   comm = xsize > ysize ? xcomm : ycomm;

273:   VecScatterInitializePackage();
274:   PetscHeaderCreate(ctx,VEC_SCATTER_CLASSID,"VecScatter","Vector Scatter","Vec",comm,VecScatterDestroy,VecScatterView);

276:   ctx->from_v  = xin;
277:   ctx->to_v    = yin;
278:   ctx->from_is = ix;
279:   ctx->to_is   = iy;
280:   VecGetLocalSize(xin,&ctx->from_n);
281:   VecGetLocalSize(yin,&ctx->to_n);

283:   /* Set default scatter type */
284:   VecScatterSetType(ctx,VECSCATTERSF);

286:   VecScatterSetFromOptions(ctx);
287:   VecScatterSetUp(ctx);

289:   *newctx = ctx;
290:   return(0);
291: }