Actual source code: openmp.c

  1: #define PETSCKSP_DLL

 3:  #include private/pcimpl.h
 4:  #include petscksp.h

  6: typedef struct {
  7:   MatStructure flag;               /* pc->flag */
  8:   PetscInt     setupcalled;        /* pc->setupcalled */
  9:   PetscInt     n;
 10:   MPI_Comm     comm;                 /* local world used by this preconditioner */
 11:   KSP          ksp;                  /* actual solver used across local world */
 12:   Mat          mat;                  /* matrix in local world */
 13:   Mat          gmat;                 /* matrix known only to process 0 in the local world */
 14:   Vec          x,y,xdummy,ydummy;
 15:   VecScatter   scatter;
 16:   PetscTruth   nonzero_guess;
 17: } PC_OpenMP;


 22: /*
 23:     Would like to have this simply call PCView() on the inner PC. The problem is
 24:   that the outter comm does not live on the inside so cannot do this. Instead 
 25:   handle the special case when the viewer is stdout, construct a new one just
 26:   for this call.
 27: */

 29: static PetscErrorCode PCView_OpenMP_MP(MPI_Comm comm,void *ctx)
 30: {
 31:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
 33:   PetscViewer    viewer;

 36:   PetscViewerASCIIGetStdout(comm,&viewer);
 37:   PetscViewerASCIIPushTab(viewer);         /* this is bogus in general */
 38:   KSPView(red->ksp,viewer);
 39:   PetscViewerASCIIPopTab(viewer);
 40:   return(0);
 41: }

 45: static PetscErrorCode PCView_OpenMP(PC pc,PetscViewer viewer)
 46: {
 47:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;
 48:   PetscMPIInt    size;
 50:   PetscTruth     iascii;


 55:   MPI_Comm_size(red->comm,&size);
 56:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 57:   if (iascii) {
 58:     PetscViewerASCIIPrintf(viewer,"  Size of solver nodes %d\n",size);
 59:     PetscViewerASCIIPrintf(viewer,"  Parallel sub-solver given next\n",size);
 60:     /* should only make the next call if the viewer is associated with stdout */
 61:     PetscOpenMPRun(red->comm,PCView_OpenMP_MP,red);
 62:   }
 63:   return(0);
 64: }


 70: static PetscErrorCode PCApply_OpenMP_1(PC pc,Vec x,Vec y)
 71: {
 72:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

 76:   KSPSetInitialGuessNonzero(red->ksp,pc->nonzero_guess);
 77:   KSPSolve(red->ksp,x,y);
 78:   return(0);
 79: }

 83: static PetscErrorCode PCSetUp_OpenMP_MP(MPI_Comm comm,void *ctx)
 84: {
 85:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
 87:   PetscInt       m;
 88:   MatReuse       scal;
 89:   PetscMPIInt    rank;

 92:   red->comm = comm;
 93:   MPI_Bcast(&red->setupcalled,1,MPIU_INT,0,comm);
 94:   MPI_Bcast(&red->flag,1,MPIU_INT,0,comm);
 95:   if (!red->setupcalled) {
 96:     /* setup vector communication */
 97:     MPI_Bcast(&red->n,1,MPIU_INT,0,comm);
 98:     VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->x);
 99:     VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->y);
100:     VecScatterCreateToZero(red->x,&red->scatter,&red->xdummy);
101:     VecDuplicate(red->xdummy,&red->ydummy);
102:     MPI_Comm_rank(comm,&rank);
103:     if (!rank) {
104:       VecDestroy(red->xdummy);
105:       VecDestroy(red->ydummy);
106:     }
107:     scal = MAT_INITIAL_MATRIX;
108:   } else {
109:     if (red->flag == DIFFERENT_NONZERO_PATTERN) {
110:       MatDestroy(red->mat);
111:       scal = MAT_INITIAL_MATRIX;
112:       CHKMEMQ;
113:     } else {
114:       scal = MAT_REUSE_MATRIX;
115:     }
116:   }

118:   /* copy matrix out onto processes */
119:   VecGetLocalSize(red->x,&m);
120:   MatDistribute_MPIAIJ(comm,red->gmat,m,scal,&red->mat);
121:   if (!red->setupcalled) {
122:     /* create the solver */
123:     KSPCreate(comm,&red->ksp);
124:     /* would like to set proper tablevel for KSP, but do not have direct access to parent pc */
125:     KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
126:     KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
127:     KSPSetFromOptions(red->ksp);
128:   } else {
129:     KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
130:   }
131:   return(0);
132: }

136: static PetscErrorCode PCSetUp_OpenMP(PC pc)
137: {
138:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;
140:   PetscMPIInt    size;

143:   red->gmat        = pc->mat;
144:   red->flag        = pc->flag;
145:   red->setupcalled = pc->setupcalled;

147:   MPI_Comm_size(red->comm,&size);
148:   if (size == 1) {  /* special case where copy of matrix is not needed */
149:     if (!red->setupcalled) {
150:       /* create the solver */
151:       KSPCreate(((PetscObject)pc)->comm,&red->ksp);
152:       PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);
153:       KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
154:       KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
155:       KSPSetFromOptions(red->ksp);
156:     } else {
157:       KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
158:     }
159:     pc->ops->apply = PCApply_OpenMP_1;
160:     return(0);
161:   } else {
162:     MatGetSize(pc->mat,&red->n,PETSC_IGNORE);
163:     PetscOpenMPRun(red->comm,PCSetUp_OpenMP_MP,red);
164:   }
165:   return(0);
166: }

170: static PetscErrorCode PCApply_OpenMP_MP(MPI_Comm comm,void *ctx)
171: {
172:   PC_OpenMP      *red = (PC_OpenMP*)ctx;

176:   VecScatterBegin(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
177:   VecScatterEnd(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
178:   MPI_Bcast(&red->nonzero_guess,1,MPIU_INT,0,red->comm);
179:   if (red->nonzero_guess) {
180:     VecScatterBegin(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
181:     VecScatterEnd(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
182:   }
183:   KSPSetInitialGuessNonzero(red->ksp,red->nonzero_guess);

185:   KSPSolve(red->ksp,red->x,red->y);

187:   VecScatterBegin(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
188:   VecScatterEnd(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
189:   return(0);
190: }

194: static PetscErrorCode PCApply_OpenMP(PC pc,Vec x,Vec y)
195: {
196:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

200:   red->xdummy        = x;
201:   red->ydummy        = y;
202:   red->nonzero_guess = pc->nonzero_guess;
203:   PetscOpenMPRun(red->comm,PCApply_OpenMP_MP,red);
204:   return(0);
205: }

209: static PetscErrorCode PCDestroy_OpenMP_MP(MPI_Comm comm,void *ctx)
210: {
211:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
212:   PetscMPIInt    rank;

216:   if (red->scatter) {VecScatterDestroy(red->scatter);}
217:   if (red->x) {VecDestroy(red->x);}
218:   if (red->y) {VecDestroy(red->y);}
219:   if (red->ksp) {KSPDestroy(red->ksp);}
220:   if (red->mat) {MatDestroy(red->mat);}
221:   MPI_Comm_rank(comm,&rank);
222:   if (rank) {
223:     if (red->xdummy) {VecDestroy(red->xdummy);}
224:     if (red->ydummy) {VecDestroy(red->ydummy);}
225:   }
226:   return(0);
227: }

231: static PetscErrorCode PCDestroy_OpenMP(PC pc)
232: {
233:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

237:   PetscOpenMPRun(red->comm,PCDestroy_OpenMP_MP,red);
238:   PetscOpenMPFree(red->comm,red);
239:   return(0);
240: }

244: static PetscErrorCode PCSetFromOptions_OpenMP(PC pc)
245: {
247:   return(0);
248: }


251: /* -------------------------------------------------------------------------------------*/
252: /*MC
253:      PCOPENMP - Runs a preconditioner for a single process matrix across several MPI processes

255: $     This will usually be run with -pc_type openmp -ksp_type preonly
256: $     solver options are set with -openmp_ksp_... and -openmp_pc_... for example
257: $     -openmp_ksp_type cg would use cg as the Krylov method or -openmp_ksp_monitor or
258: $     -openmp_pc_type hypre -openmp_pc_hypre_type boomeramg

260:        Always run with -ksp_view (or -snes_view) to see what solver is actually being used.

262:        Currently the solver options INSIDE the OpenMP preconditioner can ONLY be set via the
263:       options database.

265:    Level: intermediate

267:    See PetscOpenMPMerge() and PetscOpenMPSpawn() for two ways to start up MPI for use with this preconditioner

269: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types)

271: M*/

276: PetscErrorCode  PCCreate_OpenMP(PC pc)
277: {
279:   PC_OpenMP      *red;
280:   PetscMPIInt    size;

283:   MPI_Comm_size(((PetscObject)pc)->comm,&size);
284:   if (size > 1) SETERRQ(PETSC_ERR_ARG_SIZ,"OpenMP preconditioner only works for sequential solves");
285:   /* caste the struct length to a PetscInt for easier MPI calls */

287:   PetscOpenMPMalloc(PETSC_COMM_LOCAL_WORLD,(PetscInt)sizeof(PC_OpenMP),(void**)&red);
288:   red->comm = PETSC_COMM_LOCAL_WORLD;
289:   pc->data  = (void*) red;

291:   pc->ops->apply          = PCApply_OpenMP;
292:   pc->ops->destroy        = PCDestroy_OpenMP;
293:   pc->ops->setfromoptions = PCSetFromOptions_OpenMP;
294:   pc->ops->setup          = PCSetUp_OpenMP;
295:   pc->ops->view           = PCView_OpenMP;
296:   return(0);
297: }