Actual source code: ex3.c
petsc-3.12.5 2020-03-29
1: static char help[]= "Test PetscSFFetchAndOp on patterned SF graphs. PetscSFFetchAndOp internally uses PetscSFBcastAndOp \n\
2: and PetscSFReduce. So it is a good test to see if they all work for patterned graphs.\n\
3: Run with ./prog -op [replace | sum]\n\n";
5: #include <petscvec.h>
6: #include <petscsf.h>
7: int main(int argc,char **argv)
8: {
10: PetscInt i,N=10,low,high,nleaves;
11: PetscMPIInt size,rank;
12: Vec x,y,y2,gy2;
13: PetscScalar *rootdata,*leafdata,*leafupdate;
14: PetscLayout layout;
15: PetscSF gathersf,allgathersf,alltoallsf;
16: MPI_Op op=MPI_SUM;
17: char opname[64];
18: const char *mpiopname;
19: PetscBool flag,isreplace,issum;
21: PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
22: MPI_Comm_size(PETSC_COMM_WORLD,&size);
23: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
25: PetscOptionsGetString(NULL,NULL,"-op",opname,64,&flag);
26: PetscStrcmp(opname,"replace",&isreplace);
27: PetscStrcmp(opname,"sum",&issum);
29: if (isreplace) {op = MPIU_REPLACE; mpiopname = "MPI_REPLACE";}
30: else if (issum) {op = MPIU_SUM; mpiopname = "MPI_SUM";}
31: else SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,"Unsupported argument (%s) to -op, which must be 'replace' or 'sum'",opname);
33: VecCreate(PETSC_COMM_WORLD,&x);
34: VecSetFromOptions(x);
35: VecSetSizes(x,PETSC_DECIDE,N);
37: /*-------------------------------------*/
38: /* PETSCSF_PATTERN_GATHER */
39: /*-------------------------------------*/
41: /* set MPI vec x to [1, 2, .., N] */
42: VecGetOwnershipRange(x,&low,&high);
43: for (i=low; i<high; i++) {VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);}
44: VecAssemblyBegin(x);
45: VecAssemblyEnd(x);
47: /* Create the gather SF */
48: PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_GATHER graph with op = %s\n",mpiopname);
49: VecGetLayout(x,&layout);
50: PetscSFCreate(PETSC_COMM_WORLD,&gathersf);
51: PetscSFSetGraphWithPattern(gathersf,layout,PETSCSF_PATTERN_GATHER);
53: /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
54: PetscSFGetGraph(gathersf,NULL,&nleaves,NULL,NULL);
55: VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);
56: VecDuplicate(y,&y2);
58: VecGetArray(x,&rootdata);
59: VecGetArray(y,&leafdata);
60: VecGetArray(y2,&leafupdate);
62: /* Bcast x to y,to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */
63: PetscSFBcastAndOpBegin(gathersf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
64: PetscSFBcastAndOpEnd(gathersf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
65: VecRestoreArray(y,&leafdata);
66: VecScale(y,2);
67: VecGetArray(y,&leafdata);
69: /* FetchAndOp x to y */
70: PetscSFFetchAndOpBegin(gathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
71: PetscSFFetchAndOpEnd(gathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
73: /* View roots (x) and leafupdate (y2). Since this is a gather graph, leafudpate = rootdata = [1,N], then rootdata += leafdata, i.e., [3,3*N] */
74: VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);
75: PetscObjectSetName((PetscObject)x,"rootdata");
76: PetscObjectSetName((PetscObject)gy2,"leafupdate");
78: VecView(x,PETSC_VIEWER_STDOUT_WORLD);
79: VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);
80: VecDestroy(&gy2);
82: VecRestoreArray(y2,&leafupdate);
83: VecDestroy(&y2);
85: VecRestoreArray(y,&leafdata);
86: VecDestroy(&y);
88: VecRestoreArray(x,&rootdata);
89: /* VecDestroy(&x); */ /* We will reuse x in ALLGATHER, so do not destroy it */
91: PetscSFDestroy(&gathersf);
93: /*-------------------------------------*/
94: /* PETSCSF_PATTERN_ALLGATHER */
95: /*-------------------------------------*/
97: /* set MPI vec x to [1, 2, .., N] */
98: for (i=low; i<high; i++) {VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);}
99: VecAssemblyBegin(x);
100: VecAssemblyEnd(x);
102: /* Create the allgather SF */
103: PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLGATHER graph with op = %s\n",mpiopname);
104: VecGetLayout(x,&layout);
105: PetscSFCreate(PETSC_COMM_WORLD,&allgathersf);
106: PetscSFSetGraphWithPattern(allgathersf,layout,PETSCSF_PATTERN_ALLGATHER);
108: /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
109: PetscSFGetGraph(allgathersf,NULL,&nleaves,NULL,NULL);
110: VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);
111: VecDuplicate(y,&y2);
113: VecGetArray(x,&rootdata);
114: VecGetArray(y,&leafdata);
115: VecGetArray(y2,&leafupdate);
117: /* Bcast x to y, to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */
118: PetscSFBcastAndOpBegin(allgathersf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
119: PetscSFBcastAndOpEnd(allgathersf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
120: VecRestoreArray(y,&leafdata);
121: VecScale(y,2);
122: VecGetArray(y,&leafdata);
124: /* FetchAndOp x to y */
125: PetscSFFetchAndOpBegin(allgathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
126: PetscSFFetchAndOpEnd(allgathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
128: /* View roots (x) and leafupdate (y2). Since this is an allgather graph, we have (suppose ranks get updates in ascending order)
129: rank 0: leafupdate = rootdata = [1,N], rootdata += leafdata = [3,3*N]
130: rank 1: leafupdate = rootdata = [3,3*N], rootdata += leafdata = [5,5*N]
131: rank 2: leafupdate = rootdata = [5,5*N], rootdata += leafdata = [7,7*N]
132: ...
133: */
134: VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);
135: PetscObjectSetName((PetscObject)x,"rootdata");
136: PetscObjectSetName((PetscObject)gy2,"leafupdate");
138: VecView(x,PETSC_VIEWER_STDOUT_WORLD);
139: VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);
140: VecDestroy(&gy2);
142: VecRestoreArray(y2,&leafupdate);
143: VecDestroy(&y2);
145: VecRestoreArray(y,&leafdata);
146: VecDestroy(&y);
148: VecRestoreArray(x,&rootdata);
149: VecDestroy(&x); /* We won't reuse x in ALLGATHER, so destroy it */
151: PetscSFDestroy(&allgathersf);
153: /*-------------------------------------*/
154: /* PETSCSF_PATTERN_ALLTOALL */
155: /*-------------------------------------*/
157: VecCreate(PETSC_COMM_WORLD,&x);
158: VecSetFromOptions(x);
159: VecSetSizes(x,size,PETSC_DECIDE);
161: /* set MPI vec x to [1, 2, .., size^2] */
162: VecGetOwnershipRange(x,&low,&high);
163: for (i=low; i<high; i++) {VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);}
164: VecAssemblyBegin(x);
165: VecAssemblyEnd(x);
167: /* Create the alltoall SF */
168: PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLTOALL graph with op = %s\n",mpiopname);
169: PetscSFCreate(PETSC_COMM_WORLD,&alltoallsf);
170: PetscSFSetGraphWithPattern(alltoallsf,NULL/*insignificant*/,PETSCSF_PATTERN_ALLTOALL);
172: /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
173: PetscSFGetGraph(alltoallsf,NULL,&nleaves,NULL,NULL);
174: VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);
175: VecDuplicate(y,&y2);
177: VecGetArray(x,&rootdata);
178: VecGetArray(y,&leafdata);
179: VecGetArray(y2,&leafupdate);
181: /* Bcast x to y, to initialize y = 1+rank+size*i, with i=0..size-1 */
182: PetscSFBcastAndOpBegin(alltoallsf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
183: PetscSFBcastAndOpEnd(alltoallsf,MPIU_SCALAR,rootdata,leafdata,MPIU_REPLACE);
185: /* FetchAndOp x to y */
186: PetscSFFetchAndOpBegin(alltoallsf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
187: PetscSFFetchAndOpEnd(alltoallsf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);
189: /* View roots (x) and leafupdate (y2). Since this is an alltoall graph, each root has only one leaf.
190: So, leafupdate = rootdata = 1+rank+size*i, i=0..size-1; and rootdata += leafdata, i.e., rootdata = [2,2*N]
191: */
192: VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);
193: PetscObjectSetName((PetscObject)x,"rootdata");
194: PetscObjectSetName((PetscObject)gy2,"leafupdate");
196: VecView(x,PETSC_VIEWER_STDOUT_WORLD);
197: VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);
198: VecDestroy(&gy2);
200: VecRestoreArray(y2,&leafupdate);
201: VecDestroy(&y2);
203: VecRestoreArray(y,&leafdata);
204: VecDestroy(&y);
206: VecRestoreArray(x,&rootdata);
207: VecDestroy(&x);
209: PetscSFDestroy(&alltoallsf);
211: PetscFinalize();
212: return ierr;
213: }
215: /*TEST
217: test:
218: # N=10 is divisible by nsize, to trigger Allgather/Gather in SF
219: nsize: 2
220: args: -op replace
222: test:
223: suffix: 2
224: nsize: 2
225: args: -op sum
227: # N=10 is not divisible by nsize, to trigger Allgatherv/Gatherv in SF
228: test:
229: suffix: 3
230: nsize: 3
231: args: -op replace
233: test:
234: suffix: 4
235: nsize: 3
236: args: -op sum
238: TEST*/