Actual source code: dmmbvec.cxx

petsc-3.6.1 2015-08-06
Report Typos and Errors
  1: #include <petsc/private/dmmbimpl.h> /*I  "petscdmmoab.h"   I*/
  2: #include <petsc/private/vecimpl.h>

  4: #include <petscdmmoab.h>
  5: #include <MBTagConventions.hpp>

  7: /* declare some private DMMoab specific overrides */
  8: static PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec);
  9: static PetscErrorCode DMVecUserDestroy_Moab(void *user);
 10: static PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y);
 11: static PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name);

 15: /*@C
 16:   DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities

 18:   Collective on MPI_Comm

 20:   Input Parameter:
 21: + dm              - The DMMoab object being set
 22: . tag             - If non-zero, block size will be taken from the tag size
 23: . range           - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
 24: . is_global_vec   - If true, this is a local representation of the Vec (including ghosts in parallel), otherwise a truly parallel one
 25: . destroy_tag     - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB

 27:   Output Parameter:
 28: . vec             - The created vector

 30:   Level: beginner

 32: .keywords: Vec, create

 34: .seealso: VecCreate()
 35: @*/
 36: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,const moab::Range* range,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
 37: {
 38:   PetscErrorCode     ierr;

 41:   if(!tag && (!range || range->empty())) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Both tag and range cannot be null.");

 43:   DMCreateVector_Moab_Private(dm,tag,range,is_global_vec,destroy_tag,vec);
 44:   return(0);
 45: }


 50: /*@C
 51:   DMMoabGetVecTag - Get the MOAB tag associated with this Vec

 53:   Input Parameter:
 54: . vec - Vec being queried

 56:   Output Parameter:
 57: . tag - Tag associated with this Vec. NULL if vec is a native PETSc Vec.

 59:   Level: beginner

 61: .keywords: DMMoab, MOAB tag

 63: .seealso: DMMoabCreateVector(), DMMoabGetVecRange()
 64: @*/
 65: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
 66: {
 67:   PetscContainer  moabdata;
 68:   Vec_MOAB        *vmoab;
 69:   PetscErrorCode  ierr;


 74:   /* Get the MOAB private data */
 75:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
 76:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

 78:   *tag = vmoab->tag;
 79:   return(0);
 80: }


 85: /*@C
 86:   DMMoabGetVecRange - Get the MOAB entities associated with this Vec

 88:   Input Parameter:
 89: . vec   - Vec being queried

 91:   Output Parameter:
 92: . range - Entities associated with this Vec. NULL if vec is a native PETSc Vec.

 94:   Level: beginner

 96: .keywords: DMMoab, MOAB range

 98: .seealso: DMMoabCreateVector(), DMMoabGetVecTag()
 99: @*/
100: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
101: {
102:   PetscContainer  moabdata;
103:   Vec_MOAB        *vmoab;
104:   PetscErrorCode  ierr;


109:   /* Get the MOAB private data handle */
110:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
111:   PetscContainerGetPointer(moabdata, (void**) &vmoab);

113:   *range = *vmoab->tag_range;
114:   return(0);
115: }


120: /*@C
121:   DMMoabVecGetArray - Returns the writable direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities

123:   Collective on MPI_Comm

125:   Input Parameter:
126: . dm              - The DMMoab object being set
127: . vec             - The Vector whose underlying data is requested

129:   Output Parameter:
130: . array           - The local data array

132:   Level: intermediate

134: .keywords: MOAB, distributed array

136: .seealso: DMMoabVecRestoreArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
137: @*/
138: PetscErrorCode  DMMoabVecGetArray(DM dm,Vec vec,void* array)
139: {
140:   DM_Moab        *dmmoab;
141:   moab::ErrorCode merr;
142:   PetscErrorCode  ierr;
143:   PetscInt        count,i,f;
144:   moab::Tag       vtag;
145:   PetscScalar     **varray;
146:   PetscScalar     *marray;
147:   PetscContainer  moabdata;
148:   Vec_MOAB        *vmoab,*xmoab;

154:   dmmoab=(DM_Moab*)dm->data;

156:   /* Get the Vec_MOAB struct for the original vector */
157:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
158:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

160:   /* Get the real scalar array handle */
161:   varray = reinterpret_cast<PetscScalar**>(array);

163:   if (vmoab->is_native_vec) {

165:     /* get the local representation of the arrays from Vectors */
166:     DMGetLocalVector(dm,&vmoab->local);
167:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
168:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);

170:     /* Get the Vec_MOAB struct for the original vector */
171:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
172:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

174:     /* get the local representation of the arrays from Vectors */
175:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
176:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
177:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);

179:     VecGetArray(xmoab->local, varray);
180:   }
181:   else {

183:     /* Get the MOAB private data */
184:     DMMoabGetVecTag(vec,&vtag);

186:     /* exchange the data into ghost cells first */
187:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);

189:     PetscMalloc1((dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);

191:     /* Get the array data for local entities */
192:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
193:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

195:     i=0;
196:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
197:       for (f=0;f<dmmoab->numFields;f++,i++)
198:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
199:     }
200:   }
201:   return(0);
202: }


207: /*@C
208:   DMMoabVecRestoreArray - Restores the writable direct access array obtained via DMMoabVecGetArray

210:   Collective on MPI_Comm

212:   Input Parameter:
213: + dm              - The DMMoab object being set
214: . vec             - The Vector whose underlying data is requested
215: . array           - The local data array

217:   Level: intermediate

219: .keywords: MOAB, distributed array

221: .seealso: DMMoabVecGetArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
222: @*/
223: PetscErrorCode  DMMoabVecRestoreArray(DM dm,Vec vec,void* array)
224: {
225:   DM_Moab        *dmmoab;
226:   moab::ErrorCode merr;
227:   PetscErrorCode  ierr;
228:   moab::Tag       vtag;
229:   PetscInt        count,i,f;
230:   PetscScalar     **varray;
231:   PetscScalar     *marray;
232:   PetscContainer  moabdata;
233:   Vec_MOAB        *vmoab,*xmoab;

239:   dmmoab=(DM_Moab*)dm->data;

241:   /* Get the Vec_MOAB struct for the original vector */
242:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
243:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

245:   /* Get the real scalar array handle */
246:   varray = reinterpret_cast<PetscScalar**>(array);

248:   if (vmoab->is_native_vec) {

250:     /* Get the Vec_MOAB struct for the original vector */
251:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
252:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

254:     /* get the local representation of the arrays from Vectors */
255:     VecRestoreArray(xmoab->local, varray);
256:     VecGhostUpdateBegin(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
257:     VecGhostUpdateEnd(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
258:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);

260:     /* restore local pieces */
261:     DMLocalToGlobalBegin(dm,vmoab->local,INSERT_VALUES,vec);
262:     DMLocalToGlobalEnd(dm,vmoab->local,INSERT_VALUES,vec);
263:     DMRestoreLocalVector(dm, &vmoab->local);
264:   }
265:   else {

267:     /* Get the MOAB private data */
268:     DMMoabGetVecTag(vec,&vtag);

270:     /* Get the array data for local entities */
271:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
272:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

274:     i=0;
275:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
276:       for (f=0;f<dmmoab->numFields;f++,i++)
277:       marray[i] = (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f];
278:     }

280:     /* reduce the tags correctly -> should probably let the user choose how to reduce in the future
281:       For all FEM residual based assembly calculations, MPI_SUM should serve well */
282:     merr = dmmoab->pcomm->reduce_tags(vtag,MPI_SUM,*dmmoab->vlocal);MBERRV(dmmoab->mbiface,merr);

284:     PetscFree(*varray);
285:   }
286:   return(0);
287: }

291: /*@C
292:   DMMoabVecGetArrayRead - Returns the read-only direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities

294:   Collective on MPI_Comm

296:   Input Parameter:
297: + dm              - The DMMoab object being set
298: . vec             - The Vector whose underlying data is requested

300:   Output Parameter:
301: . array           - The local data array

303:   Level: intermediate

305: .keywords: MOAB, distributed array

307: .seealso: DMMoabVecRestoreArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
308: @*/
309: PetscErrorCode  DMMoabVecGetArrayRead(DM dm,Vec vec,void* array)
310: {
311:   DM_Moab        *dmmoab;
312:   moab::ErrorCode merr;
313:   PetscErrorCode  ierr;
314:   PetscInt        count,i,f;
315:   moab::Tag       vtag;
316:   PetscScalar     **varray;
317:   PetscScalar     *marray;
318:   PetscContainer  moabdata;
319:   Vec_MOAB        *vmoab,*xmoab;

325:   dmmoab=(DM_Moab*)dm->data;

327:   /* Get the Vec_MOAB struct for the original vector */
328:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
329:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

331:   /* Get the real scalar array handle */
332:   varray = reinterpret_cast<PetscScalar**>(array);

334:   if (vmoab->is_native_vec) {
335:     /* get the local representation of the arrays from Vectors */
336:     DMGetLocalVector(dm,&vmoab->local);
337:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
338:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);

340:     /* Get the Vec_MOAB struct for the original vector */
341:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
342:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

344:     /* get the local representation of the arrays from Vectors */
345:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
346:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
347:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
348:     VecGetArray(xmoab->local, varray);
349:   }
350:   else {
351:     /* Get the MOAB private data */
352:     DMMoabGetVecTag(vec,&vtag);

354:     /* exchange the data into ghost cells first */
355:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);

357:     PetscMalloc1((dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);

359:     /* Get the array data for local entities */
360:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
361:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);

363:     i=0;
364:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
365:       for (f=0;f<dmmoab->numFields;f++,i++)
366:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
367:     }
368:   }
369:   return(0);
370: }


375: /*@C
376:   DMMoabVecRestoreArray - Restores the read-only direct access array obtained via DMMoabVecGetArray

378:   Collective on MPI_Comm

380:   Input Parameter:
381: + dm              - The DMMoab object being set
382: . vec             - The Vector whose underlying data is requested
383: . array           - The local data array

385:   Level: intermediate

387: .keywords: MOAB, distributed array

389: .seealso: DMMoabVecGetArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
390: @*/
391: PetscErrorCode  DMMoabVecRestoreArrayRead(DM dm,Vec vec,void* array)
392: {
393:   PetscErrorCode  ierr;
394:   PetscScalar     **varray;
395:   PetscContainer  moabdata;
396:   Vec_MOAB        *vmoab,*xmoab;


403:   /* Get the Vec_MOAB struct for the original vector */
404:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
405:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

407:   /* Get the real scalar array handle */
408:   varray = reinterpret_cast<PetscScalar**>(array);

410:   if (vmoab->is_native_vec) {
411:     /* Get the Vec_MOAB struct for the original vector */
412:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
413:     PetscContainerGetPointer(moabdata, (void**)&xmoab);

415:     /* restore the local representation of the arrays from Vectors */
416:     VecRestoreArray(xmoab->local, varray);
417:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);

419:     /* restore local pieces */
420:     DMRestoreLocalVector(dm, &vmoab->local);
421:   }
422:   else {
423:     /* Nothing to do but just free the memory allocated before */
424:     PetscFree(*varray);

426:   }
427:   return(0);
428: }


433: PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
434: {
435:   PetscErrorCode    ierr;
436:   moab::ErrorCode   merr;
437:   PetscBool         is_newtag;
438:   const moab::Range *range;
439:   PetscInt          count,lnative_vec,gnative_vec;
440:   std::string       ttname;
441:   PetscScalar       *data_ptr,*defaultvals;

443:   Vec_MOAB *vmoab;
444:   DM_Moab *dmmoab = (DM_Moab*)dm->data;
445:   moab::ParallelComm *pcomm = dmmoab->pcomm;
446:   moab::Interface *mbiface = dmmoab->mbiface;

449:   if(sizeof(PetscReal) != sizeof(PetscScalar)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "MOAB tags only support Real types (Complex-type unsupported)");
450:   if(!userrange) range = dmmoab->vowned;
451:   else range = userrange;
452:   if(!range) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Input range cannot be empty or call DMSetUp first.");

454:   /* If the tag data is in a single sequence, use PETSc native vector since tag_iterate isn't useful anymore */
455:   lnative_vec=(range->psize()-1);

457: #if 0
458: //  lnative_vec=1; /* NOTE: Testing PETSc vector: will force to create native vector all the time */
459: //  lnative_vec=0; /* NOTE: Testing MOAB vector: will force to create MOAB tag_iterate based vector all the time */
460: #endif
461:   MPI_Allreduce(&lnative_vec, &gnative_vec, 1, MPI_INT, MPI_MAX, pcomm->comm());

463:   /* Create the MOAB internal data object */
464:   PetscNew(&vmoab);
465:   vmoab->is_native_vec=(gnative_vec>0?PETSC_TRUE:PETSC_FALSE);

467:   if (!vmoab->is_native_vec) {
468:     merr = mbiface->tag_get_name(tag, ttname);
469:     if (!ttname.length() && merr !=moab::MB_SUCCESS) {
470:       /* get the new name for the anonymous MOABVec -> the tag_name will be destroyed along with Tag */
471:       char *tag_name = PETSC_NULL;
472:       DMVecCreateTagName_Moab_Private(pcomm,&tag_name);
473:       is_newtag = PETSC_TRUE;

475:       /* Create the default value for the tag (all zeros) */
476:       PetscCalloc1(dmmoab->numFields,&defaultvals);

478:       /* Create the tag */
479:       merr = mbiface->tag_get_handle(tag_name,dmmoab->numFields,moab::MB_TYPE_DOUBLE,tag,
480:                                     moab::MB_TAG_DENSE|moab::MB_TAG_CREAT,defaultvals);MBERRNM(merr);
481:       PetscFree(tag_name);
482:       PetscFree(defaultvals);
483:     }
484:     else {
485:       /* Make sure the tag data is of type "double" */
486:       moab::DataType tag_type;
487:       merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
488:       if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
489:       is_newtag = destroy_tag;
490:     }

492:     vmoab->tag = tag;
493:     vmoab->new_tag = is_newtag;
494:   }
495:   vmoab->mbiface = mbiface;
496:   vmoab->pcomm = pcomm;
497:   vmoab->is_global_vec = is_global_vec;
498:   vmoab->tag_size=dmmoab->bs;
499: 
500:   if (vmoab->is_native_vec) {

502:     /* Create the PETSc Vector directly and attach our functions accordingly */
503:     if (!is_global_vec) {
504:       /* This is an MPI Vector with ghosted padding */
505:       VecCreateGhostBlock(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
506:                                  dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],vec);
507:     }
508:     else {
509:       /* This is an MPI/SEQ Vector */
510:       VecCreate(pcomm->comm(),vec);
511:       VecSetSizes(*vec,dmmoab->numFields*dmmoab->nloc,PETSC_DECIDE);
512:       VecSetBlockSize(*vec,dmmoab->bs);
513:       VecSetType(*vec, VECMPI);
514:     }
515:   }
516:   else {
517:     /* Call tag_iterate. This will cause MOAB to allocate memory for the
518:        tag data if it hasn't already happened */
519:     merr = mbiface->tag_iterate(tag,range->begin(),range->end(),count,(void*&)data_ptr);MBERRNM(merr);

521:     /* set the reference for vector range */
522:     vmoab->tag_range = new moab::Range(*range);
523:     merr = mbiface->tag_get_length(tag,dmmoab->numFields);MBERRNM(merr);

525:     /* Create the PETSc Vector
526:       Query MOAB mesh to check if there are any ghosted entities
527:         -> if we do, create a ghosted vector to map correctly to the same layout
528:         -> else, create a non-ghosted parallel vector */
529:     if (!is_global_vec) {
530:       /* This is an MPI Vector with ghosted padding */
531:       VecCreateGhostBlockWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
532:                                 dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],data_ptr,vec);
533:     }
534:     else {
535:       /* This is an MPI Vector without ghosted padding */
536:       VecCreateMPIWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*range->size(),
537:                                 PETSC_DECIDE,data_ptr,vec);
538:     }
539:   }
540:   VecSetFromOptions(*vec);

542:   /* create a container and store the internal MOAB data for faster access based on Entities etc */
543:   PetscContainer moabdata;
544:   PetscContainerCreate(PETSC_COMM_WORLD,&moabdata);
545:   PetscContainerSetPointer(moabdata,vmoab);
546:   PetscContainerSetUserDestroy(moabdata,DMVecUserDestroy_Moab);
547:   PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
548:   (*vec)->ops->duplicate = DMVecDuplicate_Moab;
549:   PetscContainerDestroy(&moabdata);

551:   /* Vector created, manually set local to global mapping */
552:   if (dmmoab->ltog_map) {
553:     VecSetLocalToGlobalMapping(*vec,dmmoab->ltog_map);
554:   }

556:   /* set the DM reference to the vector */
557:   VecSetDM(*vec, dm);
558:   return(0);
559: }


564: /*  DMVecCreateTagName_Moab_Private
565:  *
566:  *  Creates a unique tag name that will be shared across processes. If
567:  *  pcomm is NULL, then this is a serial vector. A unique tag name
568:  *  will be returned in tag_name in either case.
569:  *
570:  *  The tag names have the format _PETSC_VEC_N where N is some integer.
571:  *
572:  *  NOTE: The tag_name is allocated in this routine; The user needs to free 
573:  *        the character array.
574:  */
575: PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name)
576: {
577:   moab::ErrorCode mberr;
578:   PetscErrorCode  ierr;
579:   PetscInt        n,global_n;
580:   moab::Tag indexTag;

583:   const char*       PVEC_PREFIX      = "__PETSC_VEC_";
584:   PetscMalloc(PETSC_MAX_PATH_LEN, tag_name);

586:   /* Check to see if there are any PETSc vectors defined */
587:   moab::Interface  *mbiface = pcomm->get_moab();
588:   moab::EntityHandle rootset = mbiface->get_root_set();
589: 
590:   /* Create a tag in MOAB mesh to index and keep track of number of Petsc vec tags */
591:   mberr = mbiface->tag_get_handle("__PETSC_VECS__",1,moab::MB_TYPE_INTEGER,indexTag,
592:                                   moab::MB_TAG_SPARSE | moab::MB_TAG_CREAT,0);MBERRNM(mberr);
593:   mberr = mbiface->tag_get_data(indexTag, &rootset, 1, &n);
594:   if (mberr == moab::MB_TAG_NOT_FOUND) n=0;  /* this is the first temporary vector */
595:   else MBERRNM(mberr);

597:   /* increment the new value of n */
598:   ++n;

600:   /* Make sure that n is consistent across all processes */
601:   MPI_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,pcomm->comm());

603:   /* Set the new name accordingly and return */
604:   PetscSNPrintf(*tag_name, PETSC_MAX_PATH_LEN-1, "%s_%D", PVEC_PREFIX, global_n);
605:   mberr = mbiface->tag_set_data(indexTag, &rootset, 1, (const void*)&global_n);MBERRNM(mberr);
606:   return(0);
607: }


612: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
613: {
614:   PetscErrorCode  ierr;
615:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

620:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vowned,PETSC_TRUE,PETSC_TRUE,gvec);
621:   return(0);
622: }


627: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *lvec)
628: {
629:   PetscErrorCode  ierr;
630:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

635:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vlocal,PETSC_FALSE,PETSC_TRUE,lvec);
636:   return(0);
637: }


642: PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y)
643: {
645:   DM             dm;
646:   PetscContainer  moabdata;
647:   Vec_MOAB        *vmoab;


653:   /* Get the Vec_MOAB struct for the original vector */
654:   PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
655:   PetscContainerGetPointer(moabdata, (void**)&vmoab);

657:   VecGetDM(x, &dm);

660:   DMCreateVector_Moab_Private(dm,PETSC_NULL,vmoab->tag_range,vmoab->is_global_vec,PETSC_TRUE,y);
661:   VecSetDM(*y, dm);
662:   return(0);
663: }


668: PetscErrorCode DMVecUserDestroy_Moab(void *user)
669: {
670:   Vec_MOAB        *vmoab=(Vec_MOAB*)user;
671:   PetscErrorCode  ierr;
672:   moab::ErrorCode merr;

675:   if(vmoab->new_tag && vmoab->tag) {
676:     /* Tag was created via a call to VecDuplicate, delete the underlying tag in MOAB */
677:     merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
678:   }
679:   delete vmoab->tag_range;
680:   vmoab->tag = PETSC_NULL;
681:   vmoab->mbiface = PETSC_NULL;
682:   vmoab->pcomm = PETSC_NULL;
683:   PetscFree(vmoab);
684:   return(0);
685: }


690: PetscErrorCode  DMGlobalToLocalBegin_Moab(DM dm,Vec g,InsertMode mode,Vec l)
691: {
692:   PetscErrorCode    ierr;
693:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

696:   VecScatterBegin(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
697:   return(0);
698: }


703: PetscErrorCode  DMGlobalToLocalEnd_Moab(DM dm,Vec g,InsertMode mode,Vec l)
704: {
705:   PetscErrorCode    ierr;
706:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

709:   VecScatterEnd(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
710:   return(0);
711: }


716: PetscErrorCode  DMLocalToGlobalBegin_Moab(DM dm,Vec l,InsertMode mode,Vec g)
717: {
718:   PetscErrorCode    ierr;
719:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

722:   VecScatterBegin(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
723:   return(0);
724: }


729: PetscErrorCode  DMLocalToGlobalEnd_Moab(DM dm,Vec l,InsertMode mode,Vec g)
730: {
731:   PetscErrorCode    ierr;
732:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;

735:   VecScatterEnd(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
736:   return(0);
737: }