Actual source code: sf.c
petsc-3.5.1 2014-07-24
1: #include <petsc-private/sfimpl.h> /*I "petscsf.h" I*/
2: #include <petscctable.h>
4: /* Logging support */
5: PetscLogEvent PETSCSF_SetGraph, PETSCSF_BcastBegin, PETSCSF_BcastEnd, PETSCSF_ReduceBegin, PETSCSF_ReduceEnd, PETSCSF_FetchAndOpBegin, PETSCSF_FetchAndOpEnd;
7: #if defined(PETSC_USE_DEBUG)
8: # define PetscSFCheckGraphSet(sf,arg) do { \
9: if (PetscUnlikely(!(sf)->graphset)) \
10: SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Must call PetscSFSetGraph() on argument %D \"%s\" before %s()",(arg),#sf,PETSC_FUNCTION_NAME); \
11: } while (0)
12: #else
13: # define PetscSFCheckGraphSet(sf,arg) do {} while (0)
14: #endif
16: const char *const PetscSFDuplicateOptions[] = {"CONFONLY","RANKS","GRAPH","PetscSFDuplicateOption","PETSCSF_DUPLICATE_",0};
20: /*@C
21: PetscSFCreate - create a star forest communication context
23: Not Collective
25: Input Arguments:
26: . comm - communicator on which the star forest will operate
28: Output Arguments:
29: . sf - new star forest context
31: Level: intermediate
33: .seealso: PetscSFSetGraph(), PetscSFDestroy()
34: @*/
35: PetscErrorCode PetscSFCreate(MPI_Comm comm,PetscSF *sf)
36: {
38: PetscSF b;
42: PetscSFInitializePackage();
44: PetscHeaderCreate(b,_p_PetscSF,struct _PetscSFOps,PETSCSF_CLASSID,"PetscSF","Star Forest","PetscSF",comm,PetscSFDestroy,PetscSFView);
46: b->nroots = -1;
47: b->nleaves = -1;
48: b->nranks = -1;
49: b->rankorder = PETSC_TRUE;
50: b->ingroup = MPI_GROUP_NULL;
51: b->outgroup = MPI_GROUP_NULL;
52: b->graphset = PETSC_FALSE;
54: *sf = b;
55: return(0);
56: }
60: /*@C
61: PetscSFReset - Reset a star forest so that different sizes or neighbors can be used
63: Collective
65: Input Arguments:
66: . sf - star forest
68: Level: advanced
70: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFDestroy()
71: @*/
72: PetscErrorCode PetscSFReset(PetscSF sf)
73: {
78: sf->mine = NULL;
79: PetscFree(sf->mine_alloc);
80: sf->remote = NULL;
81: PetscFree(sf->remote_alloc);
82: PetscFree4(sf->ranks,sf->roffset,sf->rmine,sf->rremote);
83: PetscFree(sf->degree);
84: if (sf->ingroup != MPI_GROUP_NULL) {MPI_Group_free(&sf->ingroup);}
85: if (sf->outgroup != MPI_GROUP_NULL) {MPI_Group_free(&sf->outgroup);}
86: PetscSFDestroy(&sf->multi);
87: sf->graphset = PETSC_FALSE;
88: if (sf->ops->Reset) {(*sf->ops->Reset)(sf);}
89: sf->setupcalled = PETSC_FALSE;
90: return(0);
91: }
95: /*@C
96: PetscSFSetType - set the PetscSF communication implementation
98: Collective on PetscSF
100: Input Parameters:
101: + sf - the PetscSF context
102: - type - a known method
104: Options Database Key:
105: . -sf_type <type> - Sets the method; use -help for a list
106: of available methods (for instance, window, pt2pt, neighbor)
108: Notes:
109: See "include/petscsf.h" for available methods (for instance)
110: + PETSCSFWINDOW - MPI-2/3 one-sided
111: - PETSCSFBASIC - basic implementation using MPI-1 two-sided
113: Level: intermediate
115: .keywords: PetscSF, set, type
117: .seealso: PetscSFType, PetscSFCreate()
118: @*/
119: PetscErrorCode PetscSFSetType(PetscSF sf,PetscSFType type)
120: {
121: PetscErrorCode ierr,(*r)(PetscSF);
122: PetscBool match;
128: PetscObjectTypeCompare((PetscObject)sf,type,&match);
129: if (match) return(0);
131: PetscFunctionListFind(PetscSFList,type,&r);
132: if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE,"Unable to find requested PetscSF type %s",type);
133: /* Destroy the previous private PetscSF context */
134: if (sf->ops->Destroy) {
135: (*(sf)->ops->Destroy)(sf);
136: }
137: PetscMemzero(sf->ops,sizeof(*sf->ops));
138: PetscObjectChangeTypeName((PetscObject)sf,type);
139: (*r)(sf);
140: return(0);
141: }
145: /*@C
146: PetscSFDestroy - destroy star forest
148: Collective
150: Input Arguments:
151: . sf - address of star forest
153: Level: intermediate
155: .seealso: PetscSFCreate(), PetscSFReset()
156: @*/
157: PetscErrorCode PetscSFDestroy(PetscSF *sf)
158: {
162: if (!*sf) return(0);
164: if (--((PetscObject)(*sf))->refct > 0) {*sf = 0; return(0);}
165: PetscSFReset(*sf);
166: if ((*sf)->ops->Destroy) {(*(*sf)->ops->Destroy)(*sf);}
167: PetscHeaderDestroy(sf);
168: return(0);
169: }
173: /*@
174: PetscSFSetUp - set up communication structures
176: Collective
178: Input Arguments:
179: . sf - star forest communication object
181: Level: beginner
183: .seealso: PetscSFSetFromOptions(), PetscSFSetType()
184: @*/
185: PetscErrorCode PetscSFSetUp(PetscSF sf)
186: {
190: if (sf->setupcalled) return(0);
191: if (!((PetscObject)sf)->type_name) {PetscSFSetType(sf,PETSCSFBASIC);}
192: if (sf->ops->SetUp) {(*sf->ops->SetUp)(sf);}
193: sf->setupcalled = PETSC_TRUE;
194: return(0);
195: }
199: /*@C
200: PetscSFSetFromOptions - set PetscSF options using the options database
202: Logically Collective
204: Input Arguments:
205: . sf - star forest
207: Options Database Keys:
208: . -sf_synchronization - synchronization type used by PetscSF
210: Level: intermediate
212: .keywords: KSP, set, from, options, database
214: .seealso: PetscSFWindowSetSyncType()
215: @*/
216: PetscErrorCode PetscSFSetFromOptions(PetscSF sf)
217: {
218: PetscSFType deft;
219: char type[256];
221: PetscBool flg;
225: PetscObjectOptionsBegin((PetscObject)sf);
226: deft = ((PetscObject)sf)->type_name ? ((PetscObject)sf)->type_name : PETSCSFBASIC;
227: PetscOptionsFList("-sf_type","PetscSF implementation type","PetscSFSetType",PetscSFList,deft,type,256,&flg);
228: PetscSFSetType(sf,flg ? type : deft);
229: PetscOptionsBool("-sf_rank_order","sort composite points for gathers and scatters in rank order, gathers are non-deterministic otherwise","PetscSFSetRankOrder",sf->rankorder,&sf->rankorder,NULL);
230: if (sf->ops->SetFromOptions) {(*sf->ops->SetFromOptions)(sf);}
231: PetscOptionsEnd();
232: return(0);
233: }
237: /*@C
238: PetscSFSetRankOrder - sort multi-points for gathers and scatters by rank order
240: Logically Collective
242: Input Arguments:
243: + sf - star forest
244: - flg - PETSC_TRUE to sort, PETSC_FALSE to skip sorting (lower setup cost, but non-deterministic)
246: Level: advanced
248: .seealso: PetscSFGatherBegin(), PetscSFScatterBegin()
249: @*/
250: PetscErrorCode PetscSFSetRankOrder(PetscSF sf,PetscBool flg)
251: {
256: if (sf->multi) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Rank ordering must be set before first call to PetscSFGatherBegin() or PetscSFScatterBegin()");
257: sf->rankorder = flg;
258: return(0);
259: }
263: /*@C
264: PetscSFSetGraph - Set a parallel star forest
266: Collective
268: Input Arguments:
269: + sf - star forest
270: . nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
271: . nleaves - number of leaf vertices on the current process, each of these references a root on any process
272: . ilocal - locations of leaves in leafdata buffers, pass NULL for contiguous storage
273: . localmode - copy mode for ilocal
274: . iremote - remote locations of root vertices for each leaf on the current process
275: - remotemode - copy mode for iremote
277: Level: intermediate
279: .seealso: PetscSFCreate(), PetscSFView(), PetscSFGetGraph()
280: @*/
281: PetscErrorCode PetscSFSetGraph(PetscSF sf,PetscInt nroots,PetscInt nleaves,const PetscInt *ilocal,PetscCopyMode localmode,const PetscSFNode *iremote,PetscCopyMode remotemode)
282: {
283: PetscErrorCode ierr;
284: PetscTable table;
285: PetscTablePosition pos;
286: PetscMPIInt size;
287: PetscInt i,*rcount,*ranks;
291: PetscLogEventBegin(PETSCSF_SetGraph,sf,0,0,0);
294: if (nroots < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"roots %D, cannot be negative",nroots);
295: if (nleaves < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nleaves %D, cannot be negative",nleaves);
296: PetscSFReset(sf);
297: sf->nroots = nroots;
298: sf->nleaves = nleaves;
299: if (ilocal) {
300: switch (localmode) {
301: case PETSC_COPY_VALUES:
302: PetscMalloc1(nleaves,&sf->mine_alloc);
303: sf->mine = sf->mine_alloc;
304: PetscMemcpy(sf->mine,ilocal,nleaves*sizeof(*sf->mine));
305: sf->minleaf = PETSC_MAX_INT;
306: sf->maxleaf = PETSC_MIN_INT;
307: for (i=0; i<nleaves; i++) {
308: sf->minleaf = PetscMin(sf->minleaf,ilocal[i]);
309: sf->maxleaf = PetscMax(sf->maxleaf,ilocal[i]);
310: }
311: break;
312: case PETSC_OWN_POINTER:
313: sf->mine_alloc = (PetscInt*)ilocal;
314: sf->mine = sf->mine_alloc;
315: break;
316: case PETSC_USE_POINTER:
317: sf->mine = (PetscInt*)ilocal;
318: break;
319: default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown localmode");
320: }
321: }
322: if (!ilocal || nleaves > 0) {
323: sf->minleaf = 0;
324: sf->maxleaf = nleaves - 1;
325: }
326: switch (remotemode) {
327: case PETSC_COPY_VALUES:
328: PetscMalloc1(nleaves,&sf->remote_alloc);
329: sf->remote = sf->remote_alloc;
330: PetscMemcpy(sf->remote,iremote,nleaves*sizeof(*sf->remote));
331: break;
332: case PETSC_OWN_POINTER:
333: sf->remote_alloc = (PetscSFNode*)iremote;
334: sf->remote = sf->remote_alloc;
335: break;
336: case PETSC_USE_POINTER:
337: sf->remote = (PetscSFNode*)iremote;
338: break;
339: default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown remotemode");
340: }
342: MPI_Comm_size(PetscObjectComm((PetscObject)sf),&size);
343: PetscTableCreate(10,size,&table);
344: for (i=0; i<nleaves; i++) {
345: /* Log 1-based rank */
346: PetscTableAdd(table,iremote[i].rank+1,1,ADD_VALUES);
347: }
348: PetscTableGetCount(table,&sf->nranks);
349: PetscMalloc4(sf->nranks,&sf->ranks,sf->nranks+1,&sf->roffset,nleaves,&sf->rmine,nleaves,&sf->rremote);
350: PetscMalloc2(sf->nranks,&rcount,sf->nranks,&ranks);
351: PetscTableGetHeadPosition(table,&pos);
352: for (i=0; i<sf->nranks; i++) {
353: PetscTableGetNext(table,&pos,&ranks[i],&rcount[i]);
354: ranks[i]--; /* Convert back to 0-based */
355: }
356: PetscTableDestroy(&table);
357: PetscSortIntWithArray(sf->nranks,ranks,rcount);
358: sf->roffset[0] = 0;
359: for (i=0; i<sf->nranks; i++) {
360: PetscMPIIntCast(ranks[i],sf->ranks+i);
361: sf->roffset[i+1] = sf->roffset[i] + rcount[i];
362: rcount[i] = 0;
363: }
364: for (i=0; i<nleaves; i++) {
365: PetscInt lo,hi,irank;
366: /* Search for index of iremote[i].rank in sf->ranks */
367: lo = 0; hi = sf->nranks;
368: while (hi - lo > 1) {
369: PetscInt mid = lo + (hi - lo)/2;
370: if (iremote[i].rank < sf->ranks[mid]) hi = mid;
371: else lo = mid;
372: }
373: if (hi - lo == 1 && iremote[i].rank == sf->ranks[lo]) irank = lo;
374: else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Could not find rank %D in array",iremote[i].rank);
375: sf->rmine[sf->roffset[irank] + rcount[irank]] = ilocal ? ilocal[i] : i;
376: sf->rremote[sf->roffset[irank] + rcount[irank]] = iremote[i].index;
377: rcount[irank]++;
378: }
379: PetscFree2(rcount,ranks);
380: #if !defined(PETSC_USE_64BIT_INDICES)
381: if (nroots == PETSC_DETERMINE) {
382: /* Jed, if you have a better way to do this, put it in */
383: PetscInt *numRankLeaves, *leafOff, *leafIndices, *numRankRoots, *rootOff, *rootIndices, maxRoots = 0;
385: /* All to all to determine number of leaf indices from each (you can do this using Scan and asynch messages) */
386: PetscMalloc4(size,&numRankLeaves,size+1,&leafOff,size,&numRankRoots,size+1,&rootOff);
387: PetscMemzero(numRankLeaves, size * sizeof(PetscInt));
388: for (i = 0; i < nleaves; ++i) ++numRankLeaves[iremote[i].rank];
389: MPI_Alltoall(numRankLeaves, 1, MPIU_INT, numRankRoots, 1, MPIU_INT, PetscObjectComm((PetscObject)sf));
390: /* Could set nroots to this maximum */
391: for (i = 0; i < size; ++i) maxRoots += numRankRoots[i];
393: /* Gather all indices */
394: PetscMalloc2(nleaves,&leafIndices,maxRoots,&rootIndices);
395: leafOff[0] = 0;
396: for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
397: for (i = 0; i < nleaves; ++i) leafIndices[leafOff[iremote[i].rank]++] = iremote[i].index;
398: leafOff[0] = 0;
399: for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
400: rootOff[0] = 0;
401: for (i = 0; i < size; ++i) rootOff[i+1] = rootOff[i] + numRankRoots[i];
402: MPI_Alltoallv(leafIndices, numRankLeaves, leafOff, MPIU_INT, rootIndices, numRankRoots, rootOff, MPIU_INT, PetscObjectComm((PetscObject)sf));
403: /* Sort and reduce */
404: PetscSortRemoveDupsInt(&maxRoots, rootIndices);
405: PetscFree2(leafIndices,rootIndices);
406: PetscFree4(numRankLeaves,leafOff,numRankRoots,rootOff);
407: sf->nroots = maxRoots;
408: }
409: #endif
411: sf->graphset = PETSC_TRUE;
412: PetscLogEventEnd(PETSCSF_SetGraph,sf,0,0,0);
413: return(0);
414: }
418: /*@C
419: PetscSFCreateInverseSF - given a PetscSF in which all vertices have degree 1, creates the inverse map
421: Collective
423: Input Arguments:
424: . sf - star forest to invert
426: Output Arguments:
427: . isf - inverse of sf
429: Level: advanced
431: Notes:
432: All roots must have degree 1.
434: The local space may be a permutation, but cannot be sparse.
436: .seealso: PetscSFSetGraph()
437: @*/
438: PetscErrorCode PetscSFCreateInverseSF(PetscSF sf,PetscSF *isf)
439: {
441: PetscMPIInt rank;
442: PetscInt i,nroots,nleaves,maxlocal,count,*newilocal;
443: const PetscInt *ilocal;
444: PetscSFNode *roots,*leaves;
447: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
448: PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,NULL);
449: for (i=0,maxlocal=0; i<nleaves; i++) maxlocal = PetscMax(maxlocal,(ilocal ? ilocal[i] : i)+1);
450: PetscMalloc2(nroots,&roots,nleaves,&leaves);
451: for (i=0; i<nleaves; i++) {
452: leaves[i].rank = rank;
453: leaves[i].index = i;
454: }
455: for (i=0; i <nroots; i++) {
456: roots[i].rank = -1;
457: roots[i].index = -1;
458: }
459: PetscSFReduceBegin(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);
460: PetscSFReduceEnd(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);
462: /* Check whether our leaves are sparse */
463: for (i=0,count=0; i<nroots; i++) if (roots[i].rank >= 0) count++;
464: if (count == nroots) newilocal = NULL;
465: else { /* Index for sparse leaves and compact "roots" array (which is to become our leaves). */
466: PetscMalloc1(count,&newilocal);
467: for (i=0,count=0; i<nroots; i++) {
468: if (roots[i].rank >= 0) {
469: newilocal[count] = i;
470: roots[count].rank = roots[i].rank;
471: roots[count].index = roots[i].index;
472: count++;
473: }
474: }
475: }
477: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,isf);
478: PetscSFSetGraph(*isf,maxlocal,count,newilocal,PETSC_OWN_POINTER,roots,PETSC_COPY_VALUES);
479: PetscFree2(roots,leaves);
480: return(0);
481: }
485: /*@
486: PetscSFDuplicate - duplicate a PetscSF, optionally preserving rank connectivity and graph
488: Collective
490: Input Arguments:
491: + sf - communication object to duplicate
492: - opt - PETSCSF_DUPLICATE_CONFONLY, PETSCSF_DUPLICATE_RANKS, or PETSCSF_DUPLICATE_GRAPH (see PetscSFDuplicateOption)
494: Output Arguments:
495: . newsf - new communication object
497: Level: beginner
499: .seealso: PetscSFCreate(), PetscSFSetType(), PetscSFSetGraph()
500: @*/
501: PetscErrorCode PetscSFDuplicate(PetscSF sf,PetscSFDuplicateOption opt,PetscSF *newsf)
502: {
506: PetscSFCreate(PetscObjectComm((PetscObject)sf),newsf);
507: PetscSFSetType(*newsf,((PetscObject)sf)->type_name);
508: if (sf->ops->Duplicate) {(*sf->ops->Duplicate)(sf,opt,*newsf);}
509: if (opt == PETSCSF_DUPLICATE_GRAPH) {
510: PetscInt nroots,nleaves;
511: const PetscInt *ilocal;
512: const PetscSFNode *iremote;
513: PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,&iremote);
514: PetscSFSetGraph(*newsf,nroots,nleaves,ilocal,PETSC_COPY_VALUES,iremote,PETSC_COPY_VALUES);
515: }
516: return(0);
517: }
521: /*@C
522: PetscSFGetGraph - Get the graph specifying a parallel star forest
524: Not Collective
526: Input Arguments:
527: . sf - star forest
529: Output Arguments:
530: + nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
531: . nleaves - number of leaf vertices on the current process, each of these references a root on any process
532: . ilocal - locations of leaves in leafdata buffers
533: - iremote - remote locations of root vertices for each leaf on the current process
535: Level: intermediate
537: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph()
538: @*/
539: PetscErrorCode PetscSFGetGraph(PetscSF sf,PetscInt *nroots,PetscInt *nleaves,const PetscInt **ilocal,const PetscSFNode **iremote)
540: {
544: /* We are not currently requiring that the graph is set, thus returning nroots=-1 if it has not been set */
545: /* if (!sf->graphset) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Graph has not been set, must call PetscSFSetGraph()"); */
546: if (nroots) *nroots = sf->nroots;
547: if (nleaves) *nleaves = sf->nleaves;
548: if (ilocal) *ilocal = sf->mine;
549: if (iremote) *iremote = sf->remote;
550: return(0);
551: }
555: /*@C
556: PetscSFGetLeafRange - Get the active leaf ranges
558: Not Collective
560: Input Arguments:
561: . sf - star forest
563: Output Arguments:
564: + minleaf - minimum active leaf on this process
565: - maxleaf - maximum active leaf on this process
567: Level: developer
569: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph(), PetscSFGetGraph()
570: @*/
571: PetscErrorCode PetscSFGetLeafRange(PetscSF sf,PetscInt *minleaf,PetscInt *maxleaf)
572: {
576: if (minleaf) *minleaf = sf->minleaf;
577: if (maxleaf) *maxleaf = sf->maxleaf;
578: return(0);
579: }
583: /*@C
584: PetscSFView - view a star forest
586: Collective
588: Input Arguments:
589: + sf - star forest
590: - viewer - viewer to display graph, for example PETSC_VIEWER_STDOUT_WORLD
592: Level: beginner
594: .seealso: PetscSFCreate(), PetscSFSetGraph()
595: @*/
596: PetscErrorCode PetscSFView(PetscSF sf,PetscViewer viewer)
597: {
598: PetscErrorCode ierr;
599: PetscBool iascii;
600: PetscViewerFormat format;
604: if (!viewer) {PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)sf),&viewer);}
607: PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
608: if (iascii) {
609: PetscMPIInt rank;
610: PetscInt i,j;
612: PetscObjectPrintClassNamePrefixType((PetscObject)sf,viewer);
613: PetscViewerASCIIPushTab(viewer);
614: if (sf->ops->View) {(*sf->ops->View)(sf,viewer);}
615: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
616: PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
617: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of roots=%D, leaves=%D, remote ranks=%D\n",rank,sf->nroots,sf->nleaves,sf->nranks);
618: for (i=0; i<sf->nleaves; i++) {
619: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D <- (%D,%D)\n",rank,sf->mine ? sf->mine[i] : i,sf->remote[i].rank,sf->remote[i].index);
620: }
621: PetscViewerFlush(viewer);
622: PetscViewerGetFormat(viewer,&format);
623: if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
624: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Roots referenced by my leaves, by rank\n",rank);
625: for (i=0; i<sf->nranks; i++) {
626: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %d: %D edges\n",rank,sf->ranks[i],sf->roffset[i+1]-sf->roffset[i]);
627: for (j=sf->roffset[i]; j<sf->roffset[i+1]; j++) {
628: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D <- %D\n",rank,sf->rmine[j],sf->rremote[j]);
629: }
630: }
631: }
632: PetscViewerFlush(viewer);
633: PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
634: PetscViewerASCIIPopTab(viewer);
635: }
636: return(0);
637: }
641: /*@C
642: PetscSFGetRanks - Get ranks and number of vertices referenced by leaves on this process
644: Not Collective
646: Input Arguments:
647: . sf - star forest
649: Output Arguments:
650: + nranks - number of ranks referenced by local part
651: . ranks - array of ranks
652: . roffset - offset in rmine/rremote for each rank (length nranks+1)
653: . rmine - concatenated array holding local indices referencing each remote rank
654: - rremote - concatenated array holding remote indices referenced for each remote rank
656: Level: developer
658: .seealso: PetscSFSetGraph()
659: @*/
660: PetscErrorCode PetscSFGetRanks(PetscSF sf,PetscInt *nranks,const PetscMPIInt **ranks,const PetscInt **roffset,const PetscInt **rmine,const PetscInt **rremote)
661: {
665: if (nranks) *nranks = sf->nranks;
666: if (ranks) *ranks = sf->ranks;
667: if (roffset) *roffset = sf->roffset;
668: if (rmine) *rmine = sf->rmine;
669: if (rremote) *rremote = sf->rremote;
670: return(0);
671: }
675: /*@C
676: PetscSFGetGroups - gets incoming and outgoing process groups
678: Collective
680: Input Argument:
681: . sf - star forest
683: Output Arguments:
684: + incoming - group of origin processes for incoming edges (leaves that reference my roots)
685: - outgoing - group of destination processes for outgoing edges (roots that I reference)
687: Level: developer
689: .seealso: PetscSFGetWindow(), PetscSFRestoreWindow()
690: @*/
691: PetscErrorCode PetscSFGetGroups(PetscSF sf,MPI_Group *incoming,MPI_Group *outgoing)
692: {
694: MPI_Group group;
697: if (sf->ingroup == MPI_GROUP_NULL) {
698: PetscInt i;
699: const PetscInt *indegree;
700: PetscMPIInt rank,*outranks,*inranks;
701: PetscSFNode *remote;
702: PetscSF bgcount;
704: /* Compute the number of incoming ranks */
705: PetscMalloc1(sf->nranks,&remote);
706: for (i=0; i<sf->nranks; i++) {
707: remote[i].rank = sf->ranks[i];
708: remote[i].index = 0;
709: }
710: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,&bgcount);
711: PetscSFSetGraph(bgcount,1,sf->nranks,NULL,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
712: PetscSFComputeDegreeBegin(bgcount,&indegree);
713: PetscSFComputeDegreeEnd(bgcount,&indegree);
715: /* Enumerate the incoming ranks */
716: PetscMalloc2(indegree[0],&inranks,sf->nranks,&outranks);
717: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
718: for (i=0; i<sf->nranks; i++) outranks[i] = rank;
719: PetscSFGatherBegin(bgcount,MPI_INT,outranks,inranks);
720: PetscSFGatherEnd(bgcount,MPI_INT,outranks,inranks);
721: MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
722: MPI_Group_incl(group,indegree[0],inranks,&sf->ingroup);
723: MPI_Group_free(&group);
724: PetscFree2(inranks,outranks);
725: PetscSFDestroy(&bgcount);
726: }
727: *incoming = sf->ingroup;
729: if (sf->outgroup == MPI_GROUP_NULL) {
730: MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
731: MPI_Group_incl(group,sf->nranks,sf->ranks,&sf->outgroup);
732: MPI_Group_free(&group);
733: }
734: *outgoing = sf->outgroup;
735: return(0);
736: }
740: /*@C
741: PetscSFGetMultiSF - gets the inner SF implemeting gathers and scatters
743: Collective
745: Input Argument:
746: . sf - star forest that may contain roots with 0 or with more than 1 vertex
748: Output Arguments:
749: . multi - star forest with split roots, such that each root has degree exactly 1
751: Level: developer
753: Notes:
755: In most cases, users should use PetscSFGatherBegin() and PetscSFScatterBegin() instead of manipulating multi
756: directly. Since multi satisfies the stronger condition that each entry in the global space has exactly one incoming
757: edge, it is a candidate for future optimization that might involve its removal.
759: .seealso: PetscSFSetGraph(), PetscSFGatherBegin(), PetscSFScatterBegin()
760: @*/
761: PetscErrorCode PetscSFGetMultiSF(PetscSF sf,PetscSF *multi)
762: {
768: if (sf->nroots < 0) { /* Graph has not been set yet; why do we need this? */
769: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
770: *multi = sf->multi;
771: return(0);
772: }
773: if (!sf->multi) {
774: const PetscInt *indegree;
775: PetscInt i,*inoffset,*outones,*outoffset;
776: PetscSFNode *remote;
777: PetscSFComputeDegreeBegin(sf,&indegree);
778: PetscSFComputeDegreeEnd(sf,&indegree);
779: PetscMalloc3(sf->nroots+1,&inoffset,sf->nleaves,&outones,sf->nleaves,&outoffset);
780: inoffset[0] = 0;
781: #if 1
782: for (i=0; i<sf->nroots; i++) inoffset[i+1] = PetscMax(i+1, inoffset[i] + indegree[i]);
783: #else
784: for (i=0; i<sf->nroots; i++) inoffset[i+1] = inoffset[i] + indegree[i];
785: #endif
786: for (i=0; i<sf->nleaves; i++) outones[i] = 1;
787: PetscSFFetchAndOpBegin(sf,MPIU_INT,inoffset,outones,outoffset,MPIU_SUM);
788: PetscSFFetchAndOpEnd(sf,MPIU_INT,inoffset,outones,outoffset,MPIU_SUM);
789: for (i=0; i<sf->nroots; i++) inoffset[i] -= indegree[i]; /* Undo the increment */
790: #if 0
791: #if defined(PETSC_USE_DEBUG) /* Check that the expected number of increments occurred */
792: for (i=0; i<sf->nroots; i++) {
793: if (inoffset[i] + indegree[i] != inoffset[i+1]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Incorrect result after PetscSFFetchAndOp");
794: }
795: #endif
796: #endif
797: PetscMalloc1(sf->nleaves,&remote);
798: for (i=0; i<sf->nleaves; i++) {
799: remote[i].rank = sf->remote[i].rank;
800: remote[i].index = outoffset[i];
801: }
802: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
803: PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,NULL,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
804: if (sf->rankorder) { /* Sort the ranks */
805: PetscMPIInt rank;
806: PetscInt *inranks,*newoffset,*outranks,*newoutoffset,*tmpoffset,maxdegree;
807: PetscSFNode *newremote;
808: MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
809: for (i=0,maxdegree=0; i<sf->nroots; i++) maxdegree = PetscMax(maxdegree,indegree[i]);
810: PetscMalloc5(sf->multi->nroots,&inranks,sf->multi->nroots,&newoffset,sf->nleaves,&outranks,sf->nleaves,&newoutoffset,maxdegree,&tmpoffset);
811: for (i=0; i<sf->nleaves; i++) outranks[i] = rank;
812: PetscSFReduceBegin(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
813: PetscSFReduceEnd(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
814: /* Sort the incoming ranks at each vertex, build the inverse map */
815: for (i=0; i<sf->nroots; i++) {
816: PetscInt j;
817: for (j=0; j<indegree[i]; j++) tmpoffset[j] = j;
818: PetscSortIntWithArray(indegree[i],inranks+inoffset[i],tmpoffset);
819: for (j=0; j<indegree[i]; j++) newoffset[inoffset[i] + tmpoffset[j]] = inoffset[i] + j;
820: }
821: PetscSFBcastBegin(sf->multi,MPIU_INT,newoffset,newoutoffset);
822: PetscSFBcastEnd(sf->multi,MPIU_INT,newoffset,newoutoffset);
823: PetscMalloc1(sf->nleaves,&newremote);
824: for (i=0; i<sf->nleaves; i++) {
825: newremote[i].rank = sf->remote[i].rank;
826: newremote[i].index = newoutoffset[i];
827: }
828: PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,NULL,PETSC_COPY_VALUES,newremote,PETSC_OWN_POINTER);
829: PetscFree5(inranks,newoffset,outranks,newoutoffset,tmpoffset);
830: }
831: PetscFree3(inoffset,outones,outoffset);
832: }
833: *multi = sf->multi;
834: return(0);
835: }
839: /*@C
840: PetscSFCreateEmbeddedSF - removes edges from all but the selected roots, does not remap indices
842: Collective
844: Input Arguments:
845: + sf - original star forest
846: . nroots - number of roots to select on this process
847: - selected - selected roots on this process
849: Output Arguments:
850: . newsf - new star forest
852: Level: advanced
854: Note:
855: To use the new PetscSF, it may be necessary to know the indices of the leaves that are still participating. This can
856: be done by calling PetscSFGetGraph().
858: .seealso: PetscSFSetGraph(), PetscSFGetGraph()
859: @*/
860: PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF sf,PetscInt nroots,const PetscInt *selected,PetscSF *newsf)
861: {
862: PetscInt *rootdata, *leafdata, *ilocal;
863: PetscSFNode *iremote;
864: PetscInt leafsize = 0, nleaves = 0, n, i;
871: if (sf->mine) for (i = 0; i < sf->nleaves; ++i) {leafsize = PetscMax(leafsize, sf->mine[i]+1);}
872: else leafsize = sf->nleaves;
873: PetscCalloc2(sf->nroots,&rootdata,leafsize,&leafdata);
874: for (i=0; i<nroots; ++i) rootdata[selected[i]] = 1;
875: PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);
876: PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);
878: for (i = 0; i < leafsize; ++i) nleaves += leafdata[i];
879: PetscMalloc1(nleaves,&ilocal);
880: PetscMalloc1(nleaves,&iremote);
881: for (i = 0, n = 0; i < sf->nleaves; ++i) {
882: const PetscInt lidx = sf->mine ? sf->mine[i] : i;
884: if (leafdata[lidx]) {
885: ilocal[n] = lidx;
886: iremote[n].rank = sf->remote[i].rank;
887: iremote[n].index = sf->remote[i].index;
888: ++n;
889: }
890: }
891: if (n != nleaves) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "There is a size mismatch in the SF embedding, %d != %d", n, nleaves);
892: PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,newsf);
893: PetscSFSetGraph(*newsf,sf->nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);
894: PetscFree2(rootdata,leafdata);
895: return(0);
896: }
900: /*@C
901: PetscSFBcastBegin - begin pointwise broadcast to be concluded with call to PetscSFBcastEnd()
903: Collective on PetscSF
905: Input Arguments:
906: + sf - star forest on which to communicate
907: . unit - data type associated with each node
908: - rootdata - buffer to broadcast
910: Output Arguments:
911: . leafdata - buffer to update with values from each leaf's respective root
913: Level: intermediate
915: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFView(), PetscSFBcastEnd(), PetscSFReduceBegin()
916: @*/
917: PetscErrorCode PetscSFBcastBegin(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
918: {
923: PetscSFCheckGraphSet(sf,1);
924: PetscLogEventBegin(PETSCSF_BcastBegin,sf,0,0,0);
925: PetscSFSetUp(sf);
926: (*sf->ops->BcastBegin)(sf,unit,rootdata,leafdata);
927: PetscLogEventEnd(PETSCSF_BcastBegin,sf,0,0,0);
928: return(0);
929: }
933: /*@C
934: PetscSFBcastEnd - end a broadcast operation started with PetscSFBcastBegin()
936: Collective
938: Input Arguments:
939: + sf - star forest
940: . unit - data type
941: - rootdata - buffer to broadcast
943: Output Arguments:
944: . leafdata - buffer to update with values from each leaf's respective root
946: Level: intermediate
948: .seealso: PetscSFSetGraph(), PetscSFReduceEnd()
949: @*/
950: PetscErrorCode PetscSFBcastEnd(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
951: {
956: PetscSFCheckGraphSet(sf,1);
957: PetscLogEventBegin(PETSCSF_BcastEnd,sf,0,0,0);
958: PetscSFSetUp(sf);
959: (*sf->ops->BcastEnd)(sf,unit,rootdata,leafdata);
960: PetscLogEventEnd(PETSCSF_BcastEnd,sf,0,0,0);
961: return(0);
962: }
966: /*@C
967: PetscSFReduceBegin - begin reduction of leafdata into rootdata, to be completed with call to PetscSFReduceEnd()
969: Collective
971: Input Arguments:
972: + sf - star forest
973: . unit - data type
974: . leafdata - values to reduce
975: - op - reduction operation
977: Output Arguments:
978: . rootdata - result of reduction of values from all leaves of each root
980: Level: intermediate
982: .seealso: PetscSFBcastBegin()
983: @*/
984: PetscErrorCode PetscSFReduceBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
985: {
990: PetscSFCheckGraphSet(sf,1);
991: PetscLogEventBegin(PETSCSF_ReduceBegin,sf,0,0,0);
992: PetscSFSetUp(sf);
993: (sf->ops->ReduceBegin)(sf,unit,leafdata,rootdata,op);
994: PetscLogEventEnd(PETSCSF_ReduceBegin,sf,0,0,0);
995: return(0);
996: }
1000: /*@C
1001: PetscSFReduceEnd - end a reduction operation started with PetscSFReduceBegin()
1003: Collective
1005: Input Arguments:
1006: + sf - star forest
1007: . unit - data type
1008: . leafdata - values to reduce
1009: - op - reduction operation
1011: Output Arguments:
1012: . rootdata - result of reduction of values from all leaves of each root
1014: Level: intermediate
1016: .seealso: PetscSFSetGraph(), PetscSFBcastEnd()
1017: @*/
1018: PetscErrorCode PetscSFReduceEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
1019: {
1024: PetscSFCheckGraphSet(sf,1);
1025: PetscLogEventBegin(PETSCSF_ReduceEnd,sf,0,0,0);
1026: PetscSFSetUp(sf);
1027: (*sf->ops->ReduceEnd)(sf,unit,leafdata,rootdata,op);
1028: PetscLogEventEnd(PETSCSF_ReduceEnd,sf,0,0,0);
1029: return(0);
1030: }
1034: /*@C
1035: PetscSFComputeDegreeBegin - begin computation of degree for each root vertex, to be completed with PetscSFComputeDegreeEnd()
1037: Collective
1039: Input Arguments:
1040: . sf - star forest
1042: Output Arguments:
1043: . degree - degree of each root vertex
1045: Level: advanced
1047: .seealso: PetscSFGatherBegin()
1048: @*/
1049: PetscErrorCode PetscSFComputeDegreeBegin(PetscSF sf,const PetscInt **degree)
1050: {
1055: PetscSFCheckGraphSet(sf,1);
1057: if (!sf->degree) {
1058: PetscInt i;
1059: PetscMalloc1(sf->nroots,&sf->degree);
1060: PetscMalloc1(sf->nleaves,&sf->degreetmp);
1061: for (i=0; i<sf->nroots; i++) sf->degree[i] = 0;
1062: for (i=0; i<sf->nleaves; i++) sf->degreetmp[i] = 1;
1063: PetscSFReduceBegin(sf,MPIU_INT,sf->degreetmp,sf->degree,MPIU_SUM);
1064: }
1065: *degree = NULL;
1066: return(0);
1067: }
1071: /*@C
1072: PetscSFComputeDegreeEnd - complete computation of degree for each root vertex, started with PetscSFComputeDegreeBegin()
1074: Collective
1076: Input Arguments:
1077: . sf - star forest
1079: Output Arguments:
1080: . degree - degree of each root vertex
1082: Level: developer
1084: .seealso:
1085: @*/
1086: PetscErrorCode PetscSFComputeDegreeEnd(PetscSF sf,const PetscInt **degree)
1087: {
1092: PetscSFCheckGraphSet(sf,1);
1093: if (!sf->degreeknown) {
1094: PetscSFReduceEnd(sf,MPIU_INT,sf->degreetmp,sf->degree,MPIU_SUM);
1095: PetscFree(sf->degreetmp);
1097: sf->degreeknown = PETSC_TRUE;
1098: }
1099: *degree = sf->degree;
1100: return(0);
1101: }
1105: /*@C
1106: PetscSFFetchAndOpBegin - begin operation that fetches values from root and updates atomically by applying operation using my leaf value, to be completed with PetscSFFetchAndOpEnd()
1108: Collective
1110: Input Arguments:
1111: + sf - star forest
1112: . unit - data type
1113: . leafdata - leaf values to use in reduction
1114: - op - operation to use for reduction
1116: Output Arguments:
1117: + rootdata - root values to be updated, input state is seen by first process to perform an update
1118: - leafupdate - state at each leaf's respective root immediately prior to my atomic update
1120: Level: advanced
1122: Note:
1123: The update is only atomic at the granularity provided by the hardware. Different roots referenced by the same process
1124: might be updated in a different order. Furthermore, if a composite type is used for the unit datatype, atomicity is
1125: not guaranteed across the whole vertex. Therefore, this function is mostly only used with primitive types such as
1126: integers.
1128: .seealso: PetscSFComputeDegreeBegin(), PetscSFReduceBegin(), PetscSFSetGraph()
1129: @*/
1130: PetscErrorCode PetscSFFetchAndOpBegin(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1131: {
1136: PetscSFCheckGraphSet(sf,1);
1137: PetscLogEventBegin(PETSCSF_FetchAndOpBegin,sf,0,0,0);
1138: PetscSFSetUp(sf);
1139: (*sf->ops->FetchAndOpBegin)(sf,unit,rootdata,leafdata,leafupdate,op);
1140: PetscLogEventEnd(PETSCSF_FetchAndOpBegin,sf,0,0,0);
1141: return(0);
1142: }
1146: /*@C
1147: PetscSFFetchAndOpEnd - end operation started in matching call to PetscSFFetchAndOpBegin() to fetch values from roots and update atomically by applying operation using my leaf value
1149: Collective
1151: Input Arguments:
1152: + sf - star forest
1153: . unit - data type
1154: . leafdata - leaf values to use in reduction
1155: - op - operation to use for reduction
1157: Output Arguments:
1158: + rootdata - root values to be updated, input state is seen by first process to perform an update
1159: - leafupdate - state at each leaf's respective root immediately prior to my atomic update
1161: Level: advanced
1163: .seealso: PetscSFComputeDegreeEnd(), PetscSFReduceEnd(), PetscSFSetGraph()
1164: @*/
1165: PetscErrorCode PetscSFFetchAndOpEnd(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1166: {
1171: PetscSFCheckGraphSet(sf,1);
1172: PetscLogEventBegin(PETSCSF_FetchAndOpEnd,sf,0,0,0);
1173: PetscSFSetUp(sf);
1174: (*sf->ops->FetchAndOpEnd)(sf,unit,rootdata,leafdata,leafupdate,op);
1175: PetscLogEventEnd(PETSCSF_FetchAndOpEnd,sf,0,0,0);
1176: return(0);
1177: }
1181: /*@C
1182: PetscSFGatherBegin - begin pointwise gather of all leaves into multi-roots, to be completed with PetscSFGatherEnd()
1184: Collective
1186: Input Arguments:
1187: + sf - star forest
1188: . unit - data type
1189: - leafdata - leaf data to gather to roots
1191: Output Argument:
1192: . multirootdata - root buffer to gather into, amount of space per root is equal to its degree
1194: Level: intermediate
1196: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1197: @*/
1198: PetscErrorCode PetscSFGatherBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1199: {
1201: PetscSF multi;
1205: PetscSFGetMultiSF(sf,&multi);
1206: PetscSFReduceBegin(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1207: return(0);
1208: }
1212: /*@C
1213: PetscSFGatherEnd - ends pointwise gather operation that was started with PetscSFGatherBegin()
1215: Collective
1217: Input Arguments:
1218: + sf - star forest
1219: . unit - data type
1220: - leafdata - leaf data to gather to roots
1222: Output Argument:
1223: . multirootdata - root buffer to gather into, amount of space per root is equal to its degree
1225: Level: intermediate
1227: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1228: @*/
1229: PetscErrorCode PetscSFGatherEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1230: {
1232: PetscSF multi;
1236: PetscSFCheckGraphSet(sf,1);
1237: PetscSFSetUp(sf);
1238: PetscSFGetMultiSF(sf,&multi);
1239: PetscSFReduceEnd(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1240: return(0);
1241: }
1245: /*@C
1246: PetscSFScatterBegin - begin pointwise scatter operation from multi-roots to leaves, to be completed with PetscSFScatterEnd()
1248: Collective
1250: Input Arguments:
1251: + sf - star forest
1252: . unit - data type
1253: - multirootdata - root buffer to send to each leaf, one unit of data per leaf
1255: Output Argument:
1256: . leafdata - leaf data to be update with personal data from each respective root
1258: Level: intermediate
1260: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1261: @*/
1262: PetscErrorCode PetscSFScatterBegin(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1263: {
1265: PetscSF multi;
1269: PetscSFCheckGraphSet(sf,1);
1270: PetscSFSetUp(sf);
1271: PetscSFGetMultiSF(sf,&multi);
1272: PetscSFBcastBegin(multi,unit,multirootdata,leafdata);
1273: return(0);
1274: }
1278: /*@C
1279: PetscSFScatterEnd - ends pointwise scatter operation that was started with PetscSFScatterBegin()
1281: Collective
1283: Input Arguments:
1284: + sf - star forest
1285: . unit - data type
1286: - multirootdata - root buffer to send to each leaf, one unit of data per leaf
1288: Output Argument:
1289: . leafdata - leaf data to be update with personal data from each respective root
1291: Level: intermediate
1293: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1294: @*/
1295: PetscErrorCode PetscSFScatterEnd(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1296: {
1298: PetscSF multi;
1302: PetscSFCheckGraphSet(sf,1);
1303: PetscSFSetUp(sf);
1304: PetscSFGetMultiSF(sf,&multi);
1305: PetscSFBcastEnd(multi,unit,multirootdata,leafdata);
1306: return(0);
1307: }