Actual source code: vscreate.c
petsc-3.14.6 2021-03-30
1: /*
2: The VECSCATTER (vec scatter) interface routines, callable by users.
3: */
5: #include <petsc/private/vecscatterimpl.h>
7: /* Logging support */
8: PetscClassId VEC_SCATTER_CLASSID;
10: PetscFunctionList VecScatterList = NULL;
11: PetscBool VecScatterRegisterAllCalled = PETSC_FALSE;
13: /*@C
14: VecScatterSetType - Builds a vector scatter, for a particular vector scatter implementation.
16: Collective on VecScatter
18: Input Parameters:
19: + vscat - The vector scatter object
20: - type - The name of the vector scatter type
22: Options Database Key:
23: . -vecscatter_type <type> - Sets the vector scatter type; use -help for a list
24: of available types
26: Notes:
27: See "petsc/include/petscvec.h" for available vector scatter types (for instance, VECSCATTERMPI1, or VECSCATTERMPI3NODE).
29: Use VecScatterDuplicate() to form additional vectors scatter of the same type as an existing vector scatter.
31: Level: intermediate
33: .seealso: VecScatterGetType(), VecScatterCreate()
34: @*/
35: PetscErrorCode VecScatterSetType(VecScatter vscat, VecScatterType type)
36: {
37: PetscBool match;
39: PetscErrorCode (*r)(VecScatter);
43: PetscObjectTypeCompare((PetscObject)vscat, type, &match);
44: if (match) return(0);
46: PetscFunctionListFind(VecScatterList,type,&r);
47: if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE, "Unknown vector scatter type: %s",type);
49: if (vscat->ops->destroy) {
50: (*vscat->ops->destroy)(vscat);
51: vscat->ops->destroy = NULL;
52: }
54: (*r)(vscat);
55: return(0);
56: }
58: /*@C
59: VecScatterGetType - Gets the vector scatter type name (as a string) from the VecScatter.
61: Not Collective
63: Input Parameter:
64: . vscat - The vector scatter
66: Output Parameter:
67: . type - The vector scatter type name
69: Level: intermediate
71: .seealso: VecScatterSetType(), VecScatterCreate()
72: @*/
73: PetscErrorCode VecScatterGetType(VecScatter vscat, VecScatterType *type)
74: {
80: VecScatterRegisterAll();
81: *type = ((PetscObject)vscat)->type_name;
82: return(0);
83: }
85: /*@
86: VecScatterSetFromOptions - Configures the vector scatter from the options database.
88: Collective on VecScatter
90: Input Parameter:
91: . vscat - The vector scatter
93: Notes:
94: To see all options, run your program with the -help option, or consult the users manual.
95: Must be called before VecScatterSetUp() but before the vector scatter is used.
97: Level: beginner
100: .seealso: VecScatterCreate(), VecScatterDestroy(), VecScatterSetUp()
101: @*/
102: PetscErrorCode VecScatterSetFromOptions(VecScatter vscat)
103: {
105: PetscBool opt;
106: VecScatterType defaultType;
107: char typeName[256];
108: PetscMPIInt size;
113: PetscObjectOptionsBegin((PetscObject)vscat);
114: MPI_Comm_size(PetscObjectComm((PetscObject)vscat), &size);
116: /* Handle vector type options */
117: if (((PetscObject)vscat)->type_name) {
118: defaultType = ((PetscObject)vscat)->type_name;
119: } else {
120: if (size > 1) defaultType = VECSCATTERMPI1;
121: else defaultType = VECSCATTERSEQ;
122: }
124: VecScatterRegisterAll();
125: PetscOptionsFList("-vecscatter_type","Vector Scatter type","VecScatterSetType",VecScatterList,defaultType,typeName,256,&opt);
126: if (size > 1 && opt) {
127: VecScatterSetType(vscat,typeName);
128: } else {
129: VecScatterSetType(vscat,defaultType);
130: }
132: vscat->beginandendtogether = PETSC_FALSE;
133: PetscOptionsBool("-vecscatter_merge","Use combined (merged) vector scatter begin and end","VecScatterCreate",vscat->beginandendtogether,&vscat->beginandendtogether,NULL);
134: if (vscat->beginandendtogether) {
135: PetscInfo(vscat,"Using combined (merged) vector scatter begin and end\n");
136: }
138: vscat->packongpu = PETSC_TRUE;
139: PetscOptionsBool("-vecscatter_packongpu","For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI","VecScatterCreate",vscat->packongpu,&vscat->packongpu,NULL);
140: if (vscat->packongpu) {
141: PetscInfo(vscat,"For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI\n");
142: }
144: PetscOptionsEnd();
145: return(0);
146: }
148: /*@C
149: VecScatterRegister - Adds a new vector scatter component implementation
151: Not Collective
153: Input Parameters:
154: + name - The name of a new user-defined creation routine
155: - create_func - The creation routine itself
157: Notes:
158: VecScatterRegister() may be called multiple times to add several user-defined vectors
160: Sample usage:
161: .vb
162: VecScatterRegister("my_vecscatter",MyVecScatterCreate);
163: .ve
165: Then, your vector scatter type can be chosen with the procedural interface via
166: .vb
167: VecScatterCreate(MPI_Comm, VecScatter *);
168: VecScatterSetType(VecScatter,"my_vectorscatter_name");
169: .ve
170: or at runtime via the option
171: .vb
172: -vecscatter_type my_vectorscatter_name
173: .ve
175: Level: advanced
177: .seealso: VecScatterRegisterAll(), VecScatterRegisterDestroy()
178: @*/
179: PetscErrorCode VecScatterRegister(const char sname[], PetscErrorCode (*function)(VecScatter))
180: {
184: VecInitializePackage();
185: PetscFunctionListAdd(&VecScatterList,sname,function);
186: return(0);
187: }
189: /* ---------------------------------------------------------------- */
190: /*@
191: VecScatterCreate - Creates a vector scatter context.
193: Collective on Vec
195: Input Parameters:
196: + xin - a vector that defines the shape (parallel data layout of the vector)
197: of vectors from which we scatter
198: . yin - a vector that defines the shape (parallel data layout of the vector)
199: of vectors to which we scatter
200: . ix - the indices of xin to scatter (if NULL scatters all values)
201: - iy - the indices of yin to hold results (if NULL fills entire vector yin)
203: Output Parameter:
204: . newctx - location to store the new scatter context
206: Options Database Keys:
207: + -vecscatter_view - Prints detail of communications
208: . -vecscatter_view ::ascii_info - Print less details about communication
209: . -vecscatter_merge - VecScatterBegin() handles all of the communication, VecScatterEnd() is a nop
210: eliminates the chance for overlap of computation and communication
211: . -vecscatter_packtogether - Pack all messages before sending, receive all messages before unpacking
212: will make the results of scatters deterministic when otherwise they are not (it may be slower also).
213: . -vecscatter_type sf - Use the PetscSF implementation of vecscatter (Default). One can use PetscSF options to control the communication.
214: - -vecscatter_packongpu - For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI.
215: Otherwise, we might copy a segment encompassing needed entries. Default is TRUE.
217: Level: intermediate
219: Notes:
220: If both xin and yin are parallel, their communicator must be on the same
221: set of processes, but their process order can be different.
222: In calls to VecScatter() you can use different vectors than the xin and
223: yin you used above; BUT they must have the same parallel data layout, for example,
224: they could be obtained from VecDuplicate().
225: A VecScatter context CANNOT be used in two or more simultaneous scatters;
226: that is you cannot call a second VecScatterBegin() with the same scatter
227: context until the VecScatterEnd() has been called on the first VecScatterBegin().
228: In this case a separate VecScatter is needed for each concurrent scatter.
230: Currently the MPI_Send() use PERSISTENT versions.
231: (this unfortunately requires that the same in and out arrays be used for each use, this
232: is why we always need to pack the input into the work array before sending
233: and unpack upon receiving instead of using MPI datatypes to avoid the packing/unpacking).
235: Both ix and iy cannot be NULL at the same time.
237: Use VecScatterCreateToAll() to create a vecscatter that copies an MPI vector to sequential vectors on all MPI ranks.
238: Use VecScatterCreateToZero() to create a vecscatter that copies an MPI vector to a sequential vector on MPI rank 0.
239: These special vecscatters have better performance than general ones.
241: .seealso: VecScatterDestroy(), VecScatterCreateToAll(), VecScatterCreateToZero(), PetscSFCreate()
242: @*/
243: PetscErrorCode VecScatterCreate(Vec xin,IS ix,Vec yin,IS iy,VecScatter *newctx)
244: {
245: VecScatter ctx;
246: PetscErrorCode ierr;
247: PetscMPIInt xsize,ysize,result;
248: MPI_Comm comm,xcomm,ycomm;
252: *newctx = NULL;
254: if (!ix && !iy) SETERRQ(PetscObjectComm((PetscObject)xin),PETSC_ERR_SUP,"Cannot pass default in for both input and output indices");
256: /* Get comm from xin and yin */
257: PetscObjectGetComm((PetscObject)xin,&xcomm);
258: MPI_Comm_size(xcomm,&xsize);
259: PetscObjectGetComm((PetscObject)yin,&ycomm);
260: MPI_Comm_size(ycomm,&ysize);
261: if (xsize > 1 && ysize > 1) {
262: MPI_Comm_compare(xcomm,ycomm,&result);
263: if (result == MPI_UNEQUAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"VecScatterCreate: parallel vectors xin and yin must have identical/congruent/similar communicators");
264: }
266: /* If xsize==ysize, we use ycomm as comm of ctx, which does affect correctness of -vecscatter_type mpi1. But it does not
267: affect -vecscatter_type sf, since sf internally uses the roots' comm (equal to xcomm when x is parallel) as the sf
268: object's comm. One day when we degrade VecScatter from a petsc object to a simple wrapper around PetscSF, we can get
269: rid of this confusion.
270: */
271: comm = xsize > ysize ? xcomm : ycomm;
273: VecScatterInitializePackage();
274: PetscHeaderCreate(ctx,VEC_SCATTER_CLASSID,"VecScatter","Vector Scatter","Vec",comm,VecScatterDestroy,VecScatterView);
276: ctx->from_v = xin;
277: ctx->to_v = yin;
278: ctx->from_is = ix;
279: ctx->to_is = iy;
280: VecGetLocalSize(xin,&ctx->from_n);
281: VecGetLocalSize(yin,&ctx->to_n);
283: /* Set default scatter type */
284: VecScatterSetType(ctx,VECSCATTERSF);
286: VecScatterSetFromOptions(ctx);
287: VecScatterSetUp(ctx);
289: *newctx = ctx;
290: return(0);
291: }