Actual source code: dadd.c

petsc-3.14.6 2021-03-30
Report Typos and Errors
  1: #include <petsc/private/dmdaimpl.h>

  3: /*@
  4:   DMDACreatePatchIS - Creates an index set corresponding to a patch of the DA.

  6:   Not Collective

  8:   Input Parameters:
  9: +  da - the DMDA
 10: .  lower - a matstencil with i, j and k corresponding to the lower corner of the patch
 11: -  upper - a matstencil with i, j and k corresponding to the upper corner of the patch

 13:   Output Parameters:
 14: .  is - the IS corresponding to the patch

 16:   Level: developer

 18: .seealso: DMDACreateDomainDecomposition(), DMDACreateDomainDecompositionScatters()
 19: @*/
 20: PetscErrorCode DMDACreatePatchIS(DM da,MatStencil *lower,MatStencil *upper,IS *is)
 21: {
 22:   PetscInt       ms=0,ns=0,ps=0;
 23:   PetscInt       me=1,ne=1,pe=1;
 24:   PetscInt       mr=0,nr=0,pr=0;
 25:   PetscInt       ii,jj,kk;
 26:   PetscInt       si,sj,sk;
 27:   PetscInt       i,j,k,l,idx;
 28:   PetscInt       base;
 29:   PetscInt       xm=1,ym=1,zm=1;
 30:   const PetscInt *lx,*ly,*lz;
 31:   PetscInt       ox,oy,oz;
 32:   PetscInt       m,n,p,M,N,P,dof;
 33:   PetscInt       nindices;
 34:   PetscInt       *indices;
 35:   DM_DA          *dd = (DM_DA*)da->data;

 39:   /* need to get the sizes of the actual DM rather than the "global" space of a subdomain DM */
 40:   M = dd->M;N = dd->N;P=dd->P;
 41:   m = dd->m;n = dd->n;p=dd->p;
 42:   dof = dd->w;
 43:   DMDAGetOffset(da,&ox,&oy,&oz,NULL,NULL,NULL);
 44:   DMDAGetOwnershipRanges(da,&lx,&ly,&lz);
 45:   nindices = (upper->i - lower->i)*(upper->j - lower->j)*(upper->k - lower->k)*dof;
 46:   PetscMalloc1(nindices,&indices);
 47:   /* start at index 0 on processor 0 */
 48:   mr = 0;
 49:   nr = 0;
 50:   pr = 0;
 51:   ms = 0;
 52:   ns = 0;
 53:   ps = 0;
 54:   if (lx) me = lx[0];
 55:   if (ly) ne = ly[0];
 56:   if (lz) pe = lz[0];
 57:   idx = 0;
 58:   for (k=lower->k-oz;k<upper->k-oz;k++) {
 59:     for (j=lower->j-oy;j < upper->j-oy;j++) {
 60:       for (i=lower->i-ox;i < upper->i-ox;i++) {
 61:         /* "actual" indices rather than ones outside of the domain */
 62:         ii = i;
 63:         jj = j;
 64:         kk = k;
 65:         if (ii < 0) ii = M + ii;
 66:         if (jj < 0) jj = N + jj;
 67:         if (kk < 0) kk = P + kk;
 68:         if (ii > M-1) ii = ii - M;
 69:         if (jj > N-1) jj = jj - N;
 70:         if (kk > P-1) kk = kk - P;
 71:         /* gone out of processor range on x axis */
 72:         while (ii > me-1 || ii < ms) {
 73:           if (mr == m-1) {
 74:             ms = 0;
 75:             me = lx[0];
 76:             mr = 0;
 77:           } else {
 78:             mr++;
 79:             ms = me;
 80:             me += lx[mr];
 81:           }
 82:         }
 83:         /* gone out of processor range on y axis */
 84:         while (jj > ne-1 || jj < ns) {
 85:           if (nr == n-1) {
 86:             ns = 0;
 87:             ne = ly[0];
 88:             nr = 0;
 89:           } else {
 90:             nr++;
 91:             ns = ne;
 92:             ne += ly[nr];
 93:           }
 94:         }
 95:         /* gone out of processor range on z axis */
 96:         while (kk > pe-1 || kk < ps) {
 97:           if (pr == p-1) {
 98:             ps = 0;
 99:             pe = lz[0];
100:             pr = 0;
101:           } else {
102:             pr++;
103:             ps = pe;
104:             pe += lz[pr];
105:           }
106:         }
107:         /* compute the vector base on owning processor */
108:         xm = me - ms;
109:         ym = ne - ns;
110:         zm = pe - ps;
111:         base = ms*ym*zm + ns*M*zm + ps*M*N;
112:         /* compute the local coordinates on owning processor */
113:         si = ii - ms;
114:         sj = jj - ns;
115:         sk = kk - ps;
116:         for (l=0;l<dof;l++) {
117:           indices[idx] = l + dof*(base + si + xm*sj + xm*ym*sk);
118:           idx++;
119:         }
120:       }
121:     }
122:   }
123:   ISCreateGeneral(PETSC_COMM_SELF,idx,indices,PETSC_OWN_POINTER,is);
124:   return(0);
125: }

127: PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
128: {
129:   DM             *da;
130:   PetscInt       dim,size,i,j,k,idx;
132:   DMDALocalInfo  info;
133:   PetscInt       xsize,ysize,zsize;
134:   PetscInt       xo,yo,zo;
135:   PetscInt       xs,ys,zs;
136:   PetscInt       xm=1,ym=1,zm=1;
137:   PetscInt       xol,yol,zol;
138:   PetscInt       m=1,n=1,p=1;
139:   PetscInt       M,N,P;
140:   PetscInt       pm,mtmp;

143:   DMDAGetLocalInfo(dm,&info);
144:   DMDAGetOverlap(dm,&xol,&yol,&zol);
145:   DMDAGetNumLocalSubDomains(dm,&size);
146:   PetscMalloc1(size,&da);

148:   if (nlocal) *nlocal = size;

150:   dim = info.dim;

152:   M = info.xm;
153:   N = info.ym;
154:   P = info.zm;

156:   if (dim == 1) {
157:     m = size;
158:   } else if (dim == 2) {
159:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
160:     while (m > 0) {
161:       n = size/m;
162:       if (m*n*p == size) break;
163:       m--;
164:     }
165:   } else if (dim == 3) {
166:     n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));    if (!n) n = 1;
167:     while (n > 0) {
168:       pm = size/n;
169:       if (n*pm == size) break;
170:       n--;
171:     }
172:     if (!n) n = 1;
173:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
174:     if (!m) m = 1;
175:     while (m > 0) {
176:       p = size/(m*n);
177:       if (m*n*p == size) break;
178:       m--;
179:     }
180:     if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
181:   }

183:   zs = info.zs;
184:   idx = 0;
185:   for (k = 0; k < p; k++) {
186:     ys = info.ys;
187:     for (j = 0; j < n; j++) {
188:       xs = info.xs;
189:       for (i = 0; i < m; i++) {
190:         if (dim == 1) {
191:           xm = M/m + ((M % m) > i);
192:         } else if (dim == 2) {
193:           xm = M/m + ((M % m) > i);
194:           ym = N/n + ((N % n) > j);
195:         } else if (dim == 3) {
196:           xm = M/m + ((M % m) > i);
197:           ym = N/n + ((N % n) > j);
198:           zm = P/p + ((P % p) > k);
199:         }

201:         xsize = xm;
202:         ysize = ym;
203:         zsize = zm;
204:         xo = xs;
205:         yo = ys;
206:         zo = zs;

208:         DMDACreate(PETSC_COMM_SELF,&(da[idx]));
209:         DMSetOptionsPrefix(da[idx],"sub_");
210:         DMSetDimension(da[idx], info.dim);
211:         DMDASetDof(da[idx], info.dof);

213:         DMDASetStencilType(da[idx],info.st);
214:         DMDASetStencilWidth(da[idx],info.sw);

216:         if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) {
217:           xsize += xol;
218:           xo    -= xol;
219:         }
220:         if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) {
221:           ysize += yol;
222:           yo    -= yol;
223:         }
224:         if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) {
225:           zsize += zol;
226:           zo    -= zol;
227:         }

229:         if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
230:         if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
231:         if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;

233:         if (info.bx != DM_BOUNDARY_PERIODIC) {
234:           if (xo < 0) {
235:             xsize += xo;
236:             xo = 0;
237:           }
238:           if (xo+xsize > info.mx-1) {
239:             xsize -= xo+xsize - info.mx;
240:           }
241:         }
242:         if (info.by != DM_BOUNDARY_PERIODIC) {
243:           if (yo < 0) {
244:             ysize += yo;
245:             yo = 0;
246:           }
247:           if (yo+ysize > info.my-1) {
248:             ysize -= yo+ysize - info.my;
249:           }
250:         }
251:         if (info.bz != DM_BOUNDARY_PERIODIC) {
252:           if (zo < 0) {
253:             zsize += zo;
254:             zo = 0;
255:           }
256:           if (zo+zsize > info.mz-1) {
257:             zsize -= zo+zsize - info.mz;
258:           }
259:         }

261:         DMDASetSizes(da[idx], xsize, ysize, zsize);
262:         DMDASetNumProcs(da[idx], 1, 1, 1);
263:         DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);

265:         /* set up as a block instead */
266:         DMSetUp(da[idx]);

268:         /* nonoverlapping region */
269:         DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);

271:         /* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
272:         DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);
273:         xs += xm;
274:         idx++;
275:       }
276:       ys += ym;
277:     }
278:     zs += zm;
279:   }
280:   *sdm = da;
281:   return(0);
282: }

284: /*
285:  Fills the local vector problem on the subdomain from the global problem.

287:  Right now this assumes one subdomain per processor.

289:  */
290: PetscErrorCode DMCreateDomainDecompositionScatters_DA(DM dm,PetscInt nsubdms,DM *subdms,VecScatter **iscat,VecScatter **oscat, VecScatter **lscat)
291: {
293:   DMDALocalInfo  info,subinfo;
294:   DM             subdm;
295:   MatStencil     upper,lower;
296:   IS             idis,isis,odis,osis,gdis;
297:   Vec            svec,dvec,slvec;
298:   PetscInt       xm,ym,zm,xs,ys,zs;
299:   PetscInt       i;


303:   /* allocate the arrays of scatters */
304:   if (iscat) {PetscMalloc1(nsubdms,iscat);}
305:   if (oscat) {PetscMalloc1(nsubdms,oscat);}
306:   if (lscat) {PetscMalloc1(nsubdms,lscat);}

308:   DMDAGetLocalInfo(dm,&info);
309:   for (i = 0; i < nsubdms; i++) {
310:     subdm = subdms[i];
311:     DMDAGetLocalInfo(subdm,&subinfo);
312:     DMDAGetNonOverlappingRegion(subdm,&xs,&ys,&zs,&xm,&ym,&zm);

314:     /* create the global and subdomain index sets for the inner domain */
315:     lower.i = xs;
316:     lower.j = ys;
317:     lower.k = zs;
318:     upper.i = xs+xm;
319:     upper.j = ys+ym;
320:     upper.k = zs+zm;
321:     DMDACreatePatchIS(dm,&lower,&upper,&idis);
322:     DMDACreatePatchIS(subdm,&lower,&upper,&isis);

324:     /* create the global and subdomain index sets for the outer subdomain */
325:     lower.i = subinfo.xs;
326:     lower.j = subinfo.ys;
327:     lower.k = subinfo.zs;
328:     upper.i = subinfo.xs+subinfo.xm;
329:     upper.j = subinfo.ys+subinfo.ym;
330:     upper.k = subinfo.zs+subinfo.zm;
331:     DMDACreatePatchIS(dm,&lower,&upper,&odis);
332:     DMDACreatePatchIS(subdm,&lower,&upper,&osis);

334:     /* global and subdomain ISes for the local indices of the subdomain */
335:     /* todo - make this not loop over at nonperiodic boundaries, which will be more involved */
336:     lower.i = subinfo.gxs;
337:     lower.j = subinfo.gys;
338:     lower.k = subinfo.gzs;
339:     upper.i = subinfo.gxs+subinfo.gxm;
340:     upper.j = subinfo.gys+subinfo.gym;
341:     upper.k = subinfo.gzs+subinfo.gzm;

343:     DMDACreatePatchIS(dm,&lower,&upper,&gdis);

345:     /* form the scatter */
346:     DMGetGlobalVector(dm,&dvec);
347:     DMGetGlobalVector(subdm,&svec);
348:     DMGetLocalVector(subdm,&slvec);

350:     if (iscat) {VecScatterCreate(dvec,idis,svec,isis,&(*iscat)[i]);}
351:     if (oscat) {VecScatterCreate(dvec,odis,svec,osis,&(*oscat)[i]);}
352:     if (lscat) {VecScatterCreate(dvec,gdis,slvec,NULL,&(*lscat)[i]);}

354:     DMRestoreGlobalVector(dm,&dvec);
355:     DMRestoreGlobalVector(subdm,&svec);
356:     DMRestoreLocalVector(subdm,&slvec);

358:     ISDestroy(&idis);
359:     ISDestroy(&isis);

361:     ISDestroy(&odis);
362:     ISDestroy(&osis);

364:     ISDestroy(&gdis);
365:   }
366:   return(0);
367: }

369: PetscErrorCode DMDASubDomainIS_Private(DM dm,PetscInt n,DM *subdm,IS **iis,IS **ois)
370: {
372:   PetscInt       i;
373:   DMDALocalInfo  info,subinfo;
374:   MatStencil     lower,upper;

377:   DMDAGetLocalInfo(dm,&info);
378:   if (iis) {PetscMalloc1(n,iis);}
379:   if (ois) {PetscMalloc1(n,ois);}

381:   for (i = 0;i < n; i++) {
382:     DMDAGetLocalInfo(subdm[i],&subinfo);
383:     if (iis) {
384:       /* create the inner IS */
385:       lower.i = info.xs;
386:       lower.j = info.ys;
387:       lower.k = info.zs;
388:       upper.i = info.xs+info.xm;
389:       upper.j = info.ys+info.ym;
390:       upper.k = info.zs+info.zm;
391:       DMDACreatePatchIS(dm,&lower,&upper,&(*iis)[i]);
392:     }

394:     if (ois) {
395:       /* create the outer IS */
396:       lower.i = subinfo.xs;
397:       lower.j = subinfo.ys;
398:       lower.k = subinfo.zs;
399:       upper.i = subinfo.xs+subinfo.xm;
400:       upper.j = subinfo.ys+subinfo.ym;
401:       upper.k = subinfo.zs+subinfo.zm;
402:       DMDACreatePatchIS(dm,&lower,&upper,&(*ois)[i]);
403:     }
404:   }
405:   return(0);
406: }

408: PetscErrorCode DMCreateDomainDecomposition_DA(DM dm,PetscInt *len,char ***names,IS **iis,IS **ois,DM **subdm)
409: {
411:   DM             *sdm;
412:   PetscInt       n,i;

415:   DMDASubDomainDA_Private(dm,&n,&sdm);
416:   if (names) {
417:     PetscMalloc1(n,names);
418:     for (i=0;i<n;i++) (*names)[i] = NULL;
419:   }
420:   DMDASubDomainIS_Private(dm,n,sdm,iis,ois);
421:   if (subdm) *subdm = sdm;
422:   else {
423:     for (i=0;i<n;i++) {
424:       DMDestroy(&sdm[i]);
425:     }
426:   }
427:   if (len) *len = n;
428:   return(0);
429: }