Actual source code: iscoloring.c

 2:  #include petscsys.h
 3:  #include petscis.h

  7: /*@C
  8:    ISColoringDestroy - Destroys a coloring context.

 10:    Collective on ISColoring

 12:    Input Parameter:
 13: .  iscoloring - the coloring context

 15:    Level: advanced

 17: .seealso: ISColoringView(), MatGetColoring()
 18: @*/
 19: PetscErrorCode ISColoringDestroy(ISColoring iscoloring)
 20: {
 21:   PetscInt i;

 26:   if (--iscoloring->refct > 0) return(0);

 28:   if (iscoloring->is) {
 29:     for (i=0; i<iscoloring->n; i++) {
 30:       ISDestroy(iscoloring->is[i]);
 31:     }
 32:     PetscFree(iscoloring->is);
 33:   }
 34:   if (iscoloring->colors) {
 35:     PetscFree(iscoloring->colors);
 36:   }
 37:   PetscCommDestroy(&iscoloring->comm);
 38:   PetscFree(iscoloring);
 39:   return(0);
 40: }

 44: /*@C
 45:    ISColoringView - Views a coloring context.

 47:    Collective on ISColoring

 49:    Input Parameters:
 50: +  iscoloring - the coloring context
 51: -  viewer - the viewer

 53:    Level: advanced

 55: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
 56: @*/
 57: PetscErrorCode ISColoringView(ISColoring iscoloring,PetscViewer viewer)
 58: {
 59:   PetscInt       i;
 61:   PetscTruth     iascii;
 62:   IS             *is;

 66:   if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);

 69:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 70:   if (iascii) {
 71:     MPI_Comm    comm;
 72:     PetscMPIInt rank;
 73:     PetscObjectGetComm((PetscObject)viewer,&comm);
 74:     MPI_Comm_rank(comm,&rank);
 75:     PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
 76:     PetscViewerFlush(viewer);
 77:   } else {
 78:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
 79:   }

 81:   ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
 82:   for (i=0; i<iscoloring->n; i++) {
 83:     ISView(iscoloring->is[i],viewer);
 84:   }
 85:   ISColoringRestoreIS(iscoloring,&is);
 86:   return(0);
 87: }

 91: /*@C
 92:    ISColoringGetIS - Extracts index sets from the coloring context

 94:    Collective on ISColoring 

 96:    Input Parameter:
 97: .  iscoloring - the coloring context

 99:    Output Parameters:
100: +  nn - number of index sets in the coloring context
101: -  is - array of index sets

103:    Level: advanced

105: .seealso: ISColoringRestoreIS(), ISColoringView()
106: @*/
107: PetscErrorCode ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
108: {


114:   if (nn)  *nn  = iscoloring->n;
115:   if (isis) {
116:     if (!iscoloring->is) {
117:       PetscInt        *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
118:       ISColoringValue *colors = iscoloring->colors;
119:       IS              *is;
120: 
121:       /* generate the lists of nodes for each color */
122:       PetscMalloc(nc*sizeof(PetscInt),&mcolors);
123:       PetscMemzero(mcolors,nc*sizeof(PetscInt));
124:       for (i=0; i<n; i++) {
125:         mcolors[colors[i]]++;
126:       }

128:       PetscMalloc(nc*sizeof(PetscInt*),&ii);
129:       PetscMalloc(n*sizeof(PetscInt),&ii[0]);
130:       for (i=1; i<nc; i++) {
131:         ii[i] = ii[i-1] + mcolors[i-1];
132:       }
133: 
134:       MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
135:       base -= iscoloring->N;
136:       PetscMemzero(mcolors,nc*sizeof(PetscInt));
137:       for (i=0; i<n; i++) {
138:         ii[colors[i]][mcolors[colors[i]]++] = i + base;
139:       }
140:       PetscMalloc(nc*sizeof(IS),&is);
141:       for (i=0; i<nc; i++) {
142:         ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
143:       }

145:       iscoloring->is   = is;
146:       PetscFree(ii[0]);
147:       PetscFree(ii);
148:       PetscFree(mcolors);
149:     }
150:     *isis = iscoloring->is;
151:   }

153:   return(0);
154: }

158: /*@C
159:    ISColoringGetIS - Restores the index sets extracted from the coloring context

161:    Collective on ISColoring 

163:    Input Parameter:
164: +  iscoloring - the coloring context
165: -  is - array of index sets

167:    Level: advanced

169: .seealso: ISColoringGetIS(), ISColoringView()
170: @*/
171: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
172: {
175: 
176:   /* currently nothing is done here */

178:   return(0);
179: }


184: /*@C
185:     ISColoringCreate - Generates an ISColoring context from lists (provided 
186:     by each processor) of colors for each node.

188:     Collective on MPI_Comm

190:     Input Parameters:
191: +   comm - communicator for the processors creating the coloring
192: .   n - number of nodes on this processor
193: -   colors - array containing the colors for this processor, color
194:              numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
195:              and should NOT be freed (The ISColoringDestroy() will free it).

197:     Output Parameter:
198: .   iscoloring - the resulting coloring data structure

200:     Options Database Key:
201: .   -is_coloring_view - Activates ISColoringView()

203:    Level: advanced
204:    
205:     Notes: By default sets coloring type to  IS_COLORING_LOCAL

207: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()

209: @*/
210: PetscErrorCode ISColoringCreate(MPI_Comm comm,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
211: {
213:   PetscMPIInt    size,rank,tag;
214:   PetscInt       base,top,i;
215:   PetscInt       nc,ncwork;
216:   PetscTruth     flg;
217:   MPI_Status     status;

220:   PetscNew(struct _p_ISColoring,iscoloring);
221:   PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
222:   comm = (*iscoloring)->comm;

224:   /* compute the number of the first node on my processor */
225:   MPI_Comm_size(comm,&size);

227:   /* should use MPI_Scan() */
228:   MPI_Comm_rank(comm,&rank);
229:   if (!rank) {
230:     base = 0;
231:     top  = n;
232:   } else {
233:     MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
234:     top = base+n;
235:   }
236:   if (rank < size-1) {
237:     MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
238:   }

240:   /* compute the total number of colors */
241:   ncwork = 0;
242:   for (i=0; i<n; i++) {
243:     if (ncwork < colors[i]) ncwork = colors[i];
244:   }
245:   ncwork++;
246:   MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
247:   (*iscoloring)->n      = nc;
248:   (*iscoloring)->is     = 0;
249:   (*iscoloring)->colors = (ISColoringValue *)colors;
250:   (*iscoloring)->N      = n;
251:   (*iscoloring)->refct  = 1;
252:   (*iscoloring)->ctype  = IS_COLORING_LOCAL;

254:   PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
255:   if (flg) {
256:     ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
257:   }
258:   PetscLogInfo(0,"ISColoringCreate: Number of colors %d\n",nc);
259:   return(0);
260: }

264: /*@C
265:     ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
266:     generates an IS that contains a new global node number for each index based
267:     on the partitioing.

269:     Collective on IS

271:     Input Parameters
272: .   partitioning - a partitioning as generated by MatPartitioningApply()

274:     Output Parameter:
275: .   is - on each processor the index set that defines the global numbers 
276:          (in the new numbering) for all the nodes currently (before the partitioning) 
277:          on that processor

279:    Level: advanced

281: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()

283: @*/
284: PetscErrorCode ISPartitioningToNumbering(IS part,IS *is)
285: {
286:   MPI_Comm       comm;
287:   PetscInt       i,*indices,np,npt,n,*starts,*sums,*lsizes,*newi;
289:   PetscMPIInt    size;

292:   PetscObjectGetComm((PetscObject)part,&comm);
293:   MPI_Comm_size(comm,&size);

295:   /* count the number of partitions, i.e., virtual processors */
296:   ISGetLocalSize(part,&n);
297:   ISGetIndices(part,&indices);
298:   np = 0;
299:   for (i=0; i<n; i++) {
300:     np = PetscMax(np,indices[i]);
301:   }
302:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
303:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

305:   /*
306:         lsizes - number of elements of each partition on this particular processor
307:         sums - total number of "previous" nodes for any particular partition
308:         starts - global number of first element in each partition on this processor
309:   */
310:   PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
311:   PetscMemzero(lsizes,np*sizeof(PetscInt));
312:   for (i=0; i<n; i++) {
313:     lsizes[indices[i]]++;
314:   }
315:   MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
316:   MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
317:   for (i=0; i<np; i++) {
318:     starts[i] -= lsizes[i];
319:   }
320:   for (i=1; i<np; i++) {
321:     sums[i]    += sums[i-1];
322:     starts[i]  += sums[i-1];
323:   }

325:   /* 
326:       For each local index give it the new global number
327:   */
328:   PetscMalloc(n*sizeof(PetscInt),&newi);
329:   for (i=0; i<n; i++) {
330:     newi[i] = starts[indices[i]]++;
331:   }
332:   PetscFree3(lsizes,starts,sums);

334:   ISRestoreIndices(part,&indices);
335:   ISCreateGeneral(comm,n,newi,is);
336:   PetscFree(newi);
337:   ISSetPermutation(*is);
338:   return(0);
339: }

343: /*@C
344:     ISPartitioningCount - Takes a ISPartitioning and determines the number of 
345:     resulting elements on each processor

347:     Collective on IS

349:     Input Parameters:
350: .   partitioning - a partitioning as generated by MatPartitioningApply()

352:     Output Parameter:
353: .   count - array of length size, to contain the number of elements assigned
354:         to each partition, where size is the number of partitions generated
355:          (see notes below).

357:    Level: advanced

359:     Notes:
360:         By default the number of partitions generated (and thus the length
361:         of count) is the size of the communicator associated with IS,
362:         but it can be set by MatPartitioningSetNParts. The resulting array
363:         of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.


366: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
367:         MatPartitioningSetNParts()

369: @*/
370: PetscErrorCode ISPartitioningCount(IS part,PetscInt count[])
371: {
372:   MPI_Comm       comm;
373:   PetscInt            i,*indices,np,npt,n,*lsizes;
375:   PetscMPIInt    size;

378:   PetscObjectGetComm((PetscObject)part,&comm);
379:   MPI_Comm_size(comm,&size);

381:   /* count the number of partitions */
382:   ISGetLocalSize(part,&n);
383:   ISGetIndices(part,&indices);
384:   np = 0;
385:   for (i=0; i<n; i++) {
386:     np = PetscMax(np,indices[i]);
387:   }
388:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
389:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

391:   /*
392:         lsizes - number of elements of each partition on this particular processor
393:         sums - total number of "previous" nodes for any particular partition
394:         starts - global number of first element in each partition on this processor
395:   */
396:   PetscMalloc(np*sizeof(PetscInt),&lsizes);
397:   PetscMemzero(lsizes,np*sizeof(PetscInt));
398:   for (i=0; i<n; i++) {
399:     lsizes[indices[i]]++;
400:   }
401:   ISRestoreIndices(part,&indices);
402:   MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
403:   PetscFree(lsizes);
404:   return(0);
405: }

409: /*@C
410:     ISAllGather - Given an index set (IS) on each processor, generates a large 
411:     index set (same on each processor) by concatenating together each
412:     processors index set.

414:     Collective on IS

416:     Input Parameter:
417: .   is - the distributed index set

419:     Output Parameter:
420: .   isout - the concatenated index set (same on all processors)

422:     Notes: 
423:     ISAllGather() is clearly not scalable for large index sets.

425:     The IS created on each processor must be created with a common
426:     communicator (e.g., PETSC_COMM_WORLD). If the index sets were created 
427:     with PETSC_COMM_SELF, this routine will not work as expected, since 
428:     each process will generate its own new IS that consists only of
429:     itself.

431:     Level: intermediate

433:     Concepts: gather^index sets
434:     Concepts: index sets^gathering to all processors
435:     Concepts: IS^gathering to all processors

437: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
438: @*/
439: PetscErrorCode ISAllGather(IS is,IS *isout)
440: {
442:   PetscInt       *indices,n,*lindices,i,N;
443:   MPI_Comm       comm;
444:   PetscMPIInt    size,*sizes,*offsets,nn;


450:   PetscObjectGetComm((PetscObject)is,&comm);
451:   MPI_Comm_size(comm,&size);
452:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
453: 
454:   ISGetLocalSize(is,&n);
455:   nn   = (PetscMPIInt)n;
456:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
457:   offsets[0] = 0;
458:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
459:   N = offsets[size-1] + sizes[size-1];

461:   PetscMalloc(N*sizeof(PetscInt),&indices);
462:   ISGetIndices(is,&lindices);
463:   MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
464:   ISRestoreIndices(is,&lindices);
465:   PetscFree(sizes);

467:   ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
468:   PetscFree2(indices,offsets);
469:   return(0);
470: }

474: /*@C
475:     ISAllGatherIndices - Given a a set of integers on each processor, generates a large 
476:     set (same on each processor) by concatenating together each processors integers

478:     Collective on MPI_Comm

480:     Input Parameter:
481: +   comm - communicator to share the indices
482: .   n - local size of set
483: -   lindices - local indices

485:     Output Parameter:
486: +   outN - total number of indices
487: -   outindices - all of the integers

489:     Notes: 
490:     ISAllGatherIndices() is clearly not scalable for large index sets.


493:     Level: intermediate

495:     Concepts: gather^index sets
496:     Concepts: index sets^gathering to all processors
497:     Concepts: IS^gathering to all processors

499: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
500: @*/
501: PetscErrorCode ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
502: {
504:   PetscInt       *indices,i,N;
505:   PetscMPIInt    size,*sizes,*offsets,nn;

508:   MPI_Comm_size(comm,&size);
509:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
510: 
511:   nn   = n;
512:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
513:   offsets[0] = 0;
514:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
515:   N    = offsets[size-1] + sizes[size-1];

517:   PetscMalloc(N*sizeof(PetscInt),&indices);
518:   MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
519:   PetscFree2(sizes,offsets);

521:   *outindices = indices;
522:   if (outN) *outN = N;
523:   return(0);
524: }



530: /*@C
531:     ISAllGatherColors - Given a a set of colors on each processor, generates a large 
532:     set (same on each processor) by concatenating together each processors colors

534:     Collective on MPI_Comm

536:     Input Parameter:
537: +   comm - communicator to share the indices
538: .   n - local size of set
539: -   lindices - local colors

541:     Output Parameter:
542: +   outN - total number of indices
543: -   outindices - all of the colors

545:     Notes: 
546:     ISAllGatherColors() is clearly not scalable for large index sets.


549:     Level: intermediate

551:     Concepts: gather^index sets
552:     Concepts: index sets^gathering to all processors
553:     Concepts: IS^gathering to all processors

555: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
556: @*/
557: PetscErrorCode ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
558: {
559:   ISColoringValue *indices;
560:   PetscErrorCode  ierr;
561:   PetscInt        i,N;
562:   PetscMPIInt     size,*offsets,*sizes, nn = n;

565:   MPI_Comm_size(comm,&size);
566:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
567: 
568:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
569:   offsets[0] = 0;
570:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
571:   N    = offsets[size-1] + sizes[size-1];
572:   PetscFree2(sizes,offsets);

574:   PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
575:   MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);

577:   *outindices = indices;
578:   if (outN) *outN = N;
579:   return(0);
580: }