Actual source code: dmmoab.cxx
petsc-3.14.6 2021-03-30
1: #include <petsc/private/dmmbimpl.h>
3: #include <petscdmmoab.h>
4: #include <MBTagConventions.hpp>
5: #include <moab/NestedRefine.hpp>
6: #include <moab/Skinner.hpp>
8: /*MC
9: DMMOAB = "moab" - A DM object that encapsulates an unstructured mesh described by the MOAB mesh database.
10: Direct access to the MOAB Interface and other mesh manipulation related objects are available
11: through public API. Ability to create global and local representation of Vecs containing all
12: unknowns in the interior and shared boundary via a transparent tag-data wrapper is provided
13: along with utility functions to traverse the mesh and assemble a discrete system via
14: field-based/blocked Vec(Get/Set) methods. Input from and output to different formats are
15: available.
17: Reference: https://www.mcs.anl.gov/~fathom/moab-docs/html/contents.html
19: Level: intermediate
21: .seealso: DMType, DMMoabCreate(), DMCreate(), DMSetType(), DMMoabCreateMoab()
22: M*/
24: /* External function declarations here */
25: PETSC_EXTERN PetscErrorCode DMCreateInterpolation_Moab(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
26: PETSC_EXTERN PetscErrorCode DMCreateDefaultConstraints_Moab(DM dm);
27: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm, Mat *J);
28: PETSC_EXTERN PetscErrorCode DMCreateCoordinateDM_Moab(DM dm, DM *cdm);
29: PETSC_EXTERN PetscErrorCode DMRefine_Moab(DM dm, MPI_Comm comm, DM *dmRefined);
30: PETSC_EXTERN PetscErrorCode DMCoarsen_Moab(DM dm, MPI_Comm comm, DM *dmCoarsened);
31: PETSC_EXTERN PetscErrorCode DMRefineHierarchy_Moab(DM dm, PetscInt nlevels, DM dmRefined[]);
32: PETSC_EXTERN PetscErrorCode DMCoarsenHierarchy_Moab(DM dm, PetscInt nlevels, DM dmCoarsened[]);
33: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm);
34: PETSC_EXTERN PetscErrorCode DMCreateGlobalVector_Moab(DM, Vec *);
35: PETSC_EXTERN PetscErrorCode DMCreateLocalVector_Moab(DM, Vec *);
36: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm, Mat *J);
37: PETSC_EXTERN PetscErrorCode DMGlobalToLocalBegin_Moab(DM, Vec, InsertMode, Vec);
38: PETSC_EXTERN PetscErrorCode DMGlobalToLocalEnd_Moab(DM, Vec, InsertMode, Vec);
39: PETSC_EXTERN PetscErrorCode DMLocalToGlobalBegin_Moab(DM, Vec, InsertMode, Vec);
40: PETSC_EXTERN PetscErrorCode DMLocalToGlobalEnd_Moab(DM, Vec, InsertMode, Vec);
43: /* Un-implemented routines */
44: /*
45: PETSC_EXTERN PetscErrorCode DMCreatelocalsection_Moab(DM dm);
46: PETSC_EXTERN PetscErrorCode DMCreateInjection_Moab(DM dmCoarse, DM dmFine, Mat *mat);
47: PETSC_EXTERN PetscErrorCode DMLoad_Moab(DM dm, PetscViewer viewer);
48: PETSC_EXTERN PetscErrorCode DMGetDimPoints_Moab(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd);
49: PETSC_EXTERN PetscErrorCode DMCreateSubDM_Moab(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm);
50: PETSC_EXTERN PetscErrorCode DMLocatePoints_Moab(DM dm, Vec v, IS *cellIS);
51: */
53: /*@C
54: DMMoabCreate - Creates a DMMoab object, which encapsulates a moab instance
56: Collective
58: Input Parameter:
59: . comm - The communicator for the DMMoab object
61: Output Parameter:
62: . dmb - The DMMoab object
64: Level: beginner
66: @*/
67: PetscErrorCode DMMoabCreate(MPI_Comm comm, DM *dmb)
68: {
73: DMCreate(comm, dmb);
74: DMSetType(*dmb, DMMOAB);
75: return(0);
76: }
78: /*@C
79: DMMoabCreateMoab - Creates a DMMoab object, optionally from an instance and other data
81: Collective
83: Input Parameter:
84: + comm - The communicator for the DMMoab object
85: . mbiface - (ptr to) the MOAB Instance; if passed in NULL, MOAB instance is created inside PETSc, and destroyed
86: along with the DMMoab
87: . pcomm - (ptr to) a ParallelComm; if NULL, creates one internally for the whole communicator
88: . ltog_tag - A tag to use to retrieve global id for an entity; if 0, will use GLOBAL_ID_TAG_NAME/tag
89: - range - If non-NULL, contains range of entities to which DOFs will be assigned
91: Output Parameter:
92: . dmb - The DMMoab object
94: Level: intermediate
96: @*/
97: PetscErrorCode DMMoabCreateMoab(MPI_Comm comm, moab::Interface *mbiface, moab::Tag *ltog_tag, moab::Range *range, DM *dmb)
98: {
100: moab::ErrorCode merr;
101: DM dmmb;
102: DM_Moab *dmmoab;
107: DMMoabCreate(comm, &dmmb);
108: dmmoab = (DM_Moab*)(dmmb)->data;
110: if (!mbiface) {
111: dmmoab->mbiface = new moab::Core();
112: dmmoab->icreatedinstance = PETSC_TRUE;
113: }
114: else {
115: dmmoab->mbiface = mbiface;
116: dmmoab->icreatedinstance = PETSC_FALSE;
117: }
119: /* by default the fileset = root set. This set stores the hierarchy of entities belonging to current DM */
120: dmmoab->fileset = 0;
121: dmmoab->hlevel = 0;
122: dmmoab->nghostrings = 0;
124: #ifdef MOAB_HAVE_MPI
125: moab::EntityHandle partnset;
127: /* Create root sets for each mesh. Then pass these
128: to the load_file functions to be populated. */
129: merr = dmmoab->mbiface->create_meshset(moab::MESHSET_SET, partnset); MBERR("Creating partition set failed", merr);
131: /* Create the parallel communicator object with the partition handle associated with MOAB */
132: dmmoab->pcomm = moab::ParallelComm::get_pcomm(dmmoab->mbiface, partnset, &comm);
133: #endif
135: /* do the remaining initializations for DMMoab */
136: dmmoab->bs = 1;
137: dmmoab->numFields = 1;
138: PetscMalloc(dmmoab->numFields * sizeof(char*), &dmmoab->fieldNames);
139: PetscStrallocpy("DEFAULT", (char**) &dmmoab->fieldNames[0]);
140: dmmoab->rw_dbglevel = 0;
141: dmmoab->partition_by_rank = PETSC_FALSE;
142: dmmoab->extra_read_options[0] = '\0';
143: dmmoab->extra_write_options[0] = '\0';
144: dmmoab->read_mode = READ_PART;
145: dmmoab->write_mode = WRITE_PART;
147: /* set global ID tag handle */
148: if (ltog_tag && *ltog_tag) {
149: DMMoabSetLocalToGlobalTag(dmmb, *ltog_tag);
150: }
151: else {
152: merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
153: if (ltog_tag) *ltog_tag = dmmoab->ltog_tag;
154: }
156: merr = dmmoab->mbiface->tag_get_handle(MATERIAL_SET_TAG_NAME, dmmoab->material_tag); MBERRNM(merr);
158: /* set the local range of entities (vertices) of interest */
159: if (range) {
160: DMMoabSetLocalVertices(dmmb, range);
161: }
162: *dmb = dmmb;
163: return(0);
164: }
167: #ifdef MOAB_HAVE_MPI
169: /*@C
170: DMMoabGetParallelComm - Get the ParallelComm used with this DMMoab
172: Collective
174: Input Parameter:
175: . dm - The DMMoab object being set
177: Output Parameter:
178: . pcomm - The ParallelComm for the DMMoab
180: Level: beginner
182: @*/
183: PetscErrorCode DMMoabGetParallelComm(DM dm, moab::ParallelComm **pcomm)
184: {
187: *pcomm = ((DM_Moab*)(dm)->data)->pcomm;
188: return(0);
189: }
191: #endif /* MOAB_HAVE_MPI */
194: /*@C
195: DMMoabSetInterface - Set the MOAB instance used with this DMMoab
197: Collective
199: Input Parameter:
200: + dm - The DMMoab object being set
201: - mbiface - The MOAB instance being set on this DMMoab
203: Level: beginner
205: @*/
206: PetscErrorCode DMMoabSetInterface(DM dm, moab::Interface *mbiface)
207: {
208: DM_Moab *dmmoab = (DM_Moab*)(dm)->data;
213: #ifdef MOAB_HAVE_MPI
214: dmmoab->pcomm = NULL;
215: #endif
216: dmmoab->mbiface = mbiface;
217: dmmoab->icreatedinstance = PETSC_FALSE;
218: return(0);
219: }
222: /*@C
223: DMMoabGetInterface - Get the MOAB instance used with this DMMoab
225: Collective
227: Input Parameter:
228: . dm - The DMMoab object being set
230: Output Parameter:
231: . mbiface - The MOAB instance set on this DMMoab
233: Level: beginner
235: @*/
236: PetscErrorCode DMMoabGetInterface(DM dm, moab::Interface **mbiface)
237: {
238: PetscErrorCode ierr;
239: static PetscBool cite = PETSC_FALSE;
243: PetscCitationsRegister("@techreport{tautges_moab:_2004,\n type = {{SAND2004-1592}},\n title = {{MOAB:} A Mesh-Oriented Database}, institution = {Sandia National Laboratories},\n author = {Tautges, T. J. and Meyers, R. and Merkley, K. and Stimpson, C. and Ernst, C.},\n year = {2004}, note = {Report}\n}\n", &cite);
244: *mbiface = ((DM_Moab*)dm->data)->mbiface;
245: return(0);
246: }
249: /*@C
250: DMMoabSetLocalVertices - Set the entities having DOFs on this DMMoab
252: Collective
254: Input Parameter:
255: + dm - The DMMoab object being set
256: - range - The entities treated by this DMMoab
258: Level: beginner
260: @*/
261: PetscErrorCode DMMoabSetLocalVertices(DM dm, moab::Range *range)
262: {
263: moab::Range tmpvtxs;
264: DM_Moab *dmmoab = (DM_Moab*)(dm)->data;
268: dmmoab->vlocal->clear();
269: dmmoab->vowned->clear();
271: dmmoab->vlocal->insert(range->begin(), range->end());
273: #ifdef MOAB_HAVE_MPI
274: moab::ErrorCode merr;
275: /* filter based on parallel status */
276: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);
278: /* filter all the non-owned and shared entities out of the list */
279: tmpvtxs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
280: merr = dmmoab->pcomm->filter_pstatus(tmpvtxs, PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
281: tmpvtxs = moab::subtract(tmpvtxs, *dmmoab->vghost);
282: *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, tmpvtxs);
283: #else
284: *dmmoab->vowned = *dmmoab->vlocal;
285: #endif
287: /* compute and cache the sizes of local and ghosted entities */
288: dmmoab->nloc = dmmoab->vowned->size();
289: dmmoab->nghost = dmmoab->vghost->size();
290: #ifdef MOAB_HAVE_MPI
291: PetscErrorCode ierr;
292: MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
293: #else
294: dmmoab->n = dmmoab->nloc;
295: #endif
296: return(0);
297: }
300: /*@C
301: DMMoabGetAllVertices - Get the entities having DOFs on this DMMoab
303: Collective
305: Input Parameter:
306: . dm - The DMMoab object being set
308: Output Parameter:
309: . owned - The local vertex entities in this DMMoab = (owned+ghosted)
311: Level: beginner
313: @*/
314: PetscErrorCode DMMoabGetAllVertices(DM dm, moab::Range *local)
315: {
318: if (local) *local = *((DM_Moab*)dm->data)->vlocal;
319: return(0);
320: }
324: /*@C
325: DMMoabGetLocalVertices - Get the entities having DOFs on this DMMoab
327: Collective
329: Input Parameter:
330: . dm - The DMMoab object being set
332: Output Parameters:
333: + owned - The owned vertex entities in this DMMoab
334: - ghost - The ghosted entities (non-owned) stored locally in this partition
336: Level: beginner
338: @*/
339: PetscErrorCode DMMoabGetLocalVertices(DM dm, const moab::Range **owned, const moab::Range **ghost)
340: {
343: if (owned) *owned = ((DM_Moab*)dm->data)->vowned;
344: if (ghost) *ghost = ((DM_Moab*)dm->data)->vghost;
345: return(0);
346: }
348: /*@C
349: DMMoabGetLocalElements - Get the higher-dimensional entities that are locally owned
351: Collective
353: Input Parameter:
354: . dm - The DMMoab object being set
356: Output Parameter:
357: . range - The entities owned locally
359: Level: beginner
361: @*/
362: PetscErrorCode DMMoabGetLocalElements(DM dm, const moab::Range **range)
363: {
366: if (range) *range = ((DM_Moab*)dm->data)->elocal;
367: return(0);
368: }
371: /*@C
372: DMMoabSetLocalElements - Set the entities having DOFs on this DMMoab
374: Collective
376: Input Parameters:
377: + dm - The DMMoab object being set
378: - range - The entities treated by this DMMoab
380: Level: beginner
382: @*/
383: PetscErrorCode DMMoabSetLocalElements(DM dm, moab::Range *range)
384: {
385: DM_Moab *dmmoab = (DM_Moab*)(dm)->data;
389: dmmoab->elocal->clear();
390: dmmoab->eghost->clear();
391: dmmoab->elocal->insert(range->begin(), range->end());
392: #ifdef MOAB_HAVE_MPI
393: moab::ErrorCode merr;
394: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
395: *dmmoab->eghost = moab::subtract(*range, *dmmoab->elocal);
396: #endif
397: dmmoab->neleloc = dmmoab->elocal->size();
398: dmmoab->neleghost = dmmoab->eghost->size();
399: #ifdef MOAB_HAVE_MPI
400: PetscErrorCode ierr;
401: MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
402: PetscInfo2(dm, "Created %D local and %D global elements.\n", dmmoab->neleloc, dmmoab->nele);
403: #else
404: dmmoab->nele = dmmoab->neleloc;
405: #endif
406: return(0);
407: }
410: /*@C
411: DMMoabSetLocalToGlobalTag - Set the tag used for local to global numbering
413: Collective
415: Input Parameters:
416: + dm - The DMMoab object being set
417: - ltogtag - The MOAB tag used for local to global ids
419: Level: beginner
421: @*/
422: PetscErrorCode DMMoabSetLocalToGlobalTag(DM dm, moab::Tag ltogtag)
423: {
426: ((DM_Moab*)dm->data)->ltog_tag = ltogtag;
427: return(0);
428: }
431: /*@C
432: DMMoabGetLocalToGlobalTag - Get the tag used for local to global numbering
434: Collective
436: Input Parameter:
437: . dm - The DMMoab object being set
439: Output Parameter:
440: . ltogtag - The MOAB tag used for local to global ids
442: Level: beginner
444: @*/
445: PetscErrorCode DMMoabGetLocalToGlobalTag(DM dm, moab::Tag *ltog_tag)
446: {
449: *ltog_tag = ((DM_Moab*)dm->data)->ltog_tag;
450: return(0);
451: }
454: /*@C
455: DMMoabSetBlockSize - Set the block size used with this DMMoab
457: Collective
459: Input Parameter:
460: + dm - The DMMoab object being set
461: - bs - The block size used with this DMMoab
463: Level: beginner
465: @*/
466: PetscErrorCode DMMoabSetBlockSize(DM dm, PetscInt bs)
467: {
470: ((DM_Moab*)dm->data)->bs = bs;
471: return(0);
472: }
475: /*@C
476: DMMoabGetBlockSize - Get the block size used with this DMMoab
478: Collective
480: Input Parameter:
481: . dm - The DMMoab object being set
483: Output Parameter:
484: . bs - The block size used with this DMMoab
486: Level: beginner
488: @*/
489: PetscErrorCode DMMoabGetBlockSize(DM dm, PetscInt *bs)
490: {
493: *bs = ((DM_Moab*)dm->data)->bs;
494: return(0);
495: }
498: /*@C
499: DMMoabGetSize - Get the global vertex size used with this DMMoab
501: Collective on dm
503: Input Parameter:
504: . dm - The DMMoab object being set
506: Output Parameter:
507: + neg - The number of global elements in the DMMoab instance
508: - nvg - The number of global vertices in the DMMoab instance
510: Level: beginner
512: @*/
513: PetscErrorCode DMMoabGetSize(DM dm, PetscInt *neg, PetscInt *nvg)
514: {
517: if (neg) *neg = ((DM_Moab*)dm->data)->nele;
518: if (nvg) *nvg = ((DM_Moab*)dm->data)->n;
519: return(0);
520: }
523: /*@C
524: DMMoabGetLocalSize - Get the local and ghosted vertex size used with this DMMoab
526: Collective on dm
528: Input Parameter:
529: . dm - The DMMoab object being set
531: Output Parameter:
532: + nel - The number of owned elements in this processor
533: . neg - The number of ghosted elements in this processor
534: . nvl - The number of owned vertices in this processor
535: - nvg - The number of ghosted vertices in this processor
537: Level: beginner
539: @*/
540: PetscErrorCode DMMoabGetLocalSize(DM dm, PetscInt *nel, PetscInt *neg, PetscInt *nvl, PetscInt *nvg)
541: {
544: if (nel) *nel = ((DM_Moab*)dm->data)->neleloc;
545: if (neg) *neg = ((DM_Moab*)dm->data)->neleghost;
546: if (nvl) *nvl = ((DM_Moab*)dm->data)->nloc;
547: if (nvg) *nvg = ((DM_Moab*)dm->data)->nghost;
548: return(0);
549: }
552: /*@C
553: DMMoabGetOffset - Get the local offset for the global vector
555: Collective
557: Input Parameter:
558: . dm - The DMMoab object being set
560: Output Parameter:
561: . offset - The local offset for the global vector
563: Level: beginner
565: @*/
566: PetscErrorCode DMMoabGetOffset(DM dm, PetscInt *offset)
567: {
570: *offset = ((DM_Moab*)dm->data)->vstart;
571: return(0);
572: }
575: /*@C
576: DMMoabGetDimension - Get the dimension of the DM Mesh
578: Collective
580: Input Parameter:
581: . dm - The DMMoab object
583: Output Parameter:
584: . dim - The dimension of DM
586: Level: beginner
588: @*/
589: PetscErrorCode DMMoabGetDimension(DM dm, PetscInt *dim)
590: {
593: *dim = ((DM_Moab*)dm->data)->dim;
594: return(0);
595: }
598: /*@C
599: DMMoabGetHierarchyLevel - Get the current level of the mesh hierarchy
600: generated through uniform refinement.
602: Collective on dm
604: Input Parameter:
605: . dm - The DMMoab object being set
607: Output Parameter:
608: . nvg - The current mesh hierarchy level
610: Level: beginner
612: @*/
613: PetscErrorCode DMMoabGetHierarchyLevel(DM dm, PetscInt *nlevel)
614: {
617: if (nlevel) *nlevel = ((DM_Moab*)dm->data)->hlevel;
618: return(0);
619: }
622: /*@C
623: DMMoabGetMaterialBlock - Get the material ID corresponding to the current entity of the DM Mesh
625: Collective
627: Input Parameter:
628: + dm - The DMMoab object
629: - ehandle - The element entity handle
631: Output Parameter:
632: . mat - The material ID for the current entity
634: Level: beginner
636: @*/
637: PetscErrorCode DMMoabGetMaterialBlock(DM dm, const moab::EntityHandle ehandle, PetscInt *mat)
638: {
639: DM_Moab *dmmoab;
643: if (*mat) {
644: dmmoab = (DM_Moab*)(dm)->data;
645: *mat = dmmoab->materials[dmmoab->elocal->index(ehandle)];
646: }
647: return(0);
648: }
651: /*@C
652: DMMoabGetVertexCoordinates - Get the coordinates corresponding to the requested vertex entities
654: Collective
656: Input Parameter:
657: + dm - The DMMoab object
658: . nconn - Number of entities whose coordinates are needed
659: - conn - The vertex entity handles
661: Output Parameter:
662: . vpos - The coordinates of the requested vertex entities
664: Level: beginner
666: .seealso: DMMoabGetVertexConnectivity()
667: @*/
668: PetscErrorCode DMMoabGetVertexCoordinates(DM dm, PetscInt nconn, const moab::EntityHandle *conn, PetscReal *vpos)
669: {
670: DM_Moab *dmmoab;
671: moab::ErrorCode merr;
677: dmmoab = (DM_Moab*)(dm)->data;
679: /* Get connectivity information in MOAB canonical ordering */
680: if (dmmoab->hlevel) {
681: merr = dmmoab->hierarchy->get_coordinates(const_cast<moab::EntityHandle*>(conn), nconn, dmmoab->hlevel, vpos);MBERRNM(merr);
682: }
683: else {
684: merr = dmmoab->mbiface->get_coords(conn, nconn, vpos);MBERRNM(merr);
685: }
686: return(0);
687: }
690: /*@C
691: DMMoabGetVertexConnectivity - Get the vertex adjacency for the given entity
693: Collective
695: Input Parameter:
696: + dm - The DMMoab object
697: - vhandle - Vertex entity handle
699: Output Parameter:
700: + nconn - Number of entities whose coordinates are needed
701: - conn - The vertex entity handles
703: Level: beginner
705: .seealso: DMMoabGetVertexCoordinates(), DMMoabRestoreVertexConnectivity()
706: @*/
707: PetscErrorCode DMMoabGetVertexConnectivity(DM dm, moab::EntityHandle vhandle, PetscInt* nconn, moab::EntityHandle **conn)
708: {
709: DM_Moab *dmmoab;
710: std::vector<moab::EntityHandle> adj_entities, connect;
711: PetscErrorCode ierr;
712: moab::ErrorCode merr;
717: dmmoab = (DM_Moab*)(dm)->data;
719: /* Get connectivity information in MOAB canonical ordering */
720: merr = dmmoab->mbiface->get_adjacencies(&vhandle, 1, 1, true, adj_entities, moab::Interface::UNION); MBERRNM(merr);
721: merr = dmmoab->mbiface->get_connectivity(&adj_entities[0], adj_entities.size(), connect); MBERRNM(merr);
723: if (conn) {
724: PetscMalloc(sizeof(moab::EntityHandle) * connect.size(), conn);
725: PetscArraycpy(*conn, &connect[0], connect.size());
726: }
727: if (nconn) *nconn = connect.size();
728: return(0);
729: }
732: /*@C
733: DMMoabRestoreVertexConnectivity - Restore the vertex connectivity for the given entity
735: Collective
737: Input Parameter:
738: + dm - The DMMoab object
739: . vhandle - Vertex entity handle
740: . nconn - Number of entities whose coordinates are needed
741: - conn - The vertex entity handles
743: Level: beginner
745: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity()
746: @*/
747: PetscErrorCode DMMoabRestoreVertexConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, moab::EntityHandle **conn)
748: {
749: PetscErrorCode ierr;
755: if (conn) {
756: PetscFree(*conn);
757: }
758: if (nconn) *nconn = 0;
759: return(0);
760: }
763: /*@C
764: DMMoabGetElementConnectivity - Get the vertex adjacency for the given entity
766: Collective
768: Input Parameter:
769: + dm - The DMMoab object
770: - ehandle - Vertex entity handle
772: Output Parameter:
773: + nconn - Number of entities whose coordinates are needed
774: - conn - The vertex entity handles
776: Level: beginner
778: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity(), DMMoabRestoreVertexConnectivity()
779: @*/
780: PetscErrorCode DMMoabGetElementConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, const moab::EntityHandle **conn)
781: {
782: DM_Moab *dmmoab;
783: const moab::EntityHandle *connect;
784: std::vector<moab::EntityHandle> vconn;
785: moab::ErrorCode merr;
786: PetscInt nnodes;
791: dmmoab = (DM_Moab*)(dm)->data;
793: /* Get connectivity information in MOAB canonical ordering */
794: merr = dmmoab->mbiface->get_connectivity(ehandle, connect, nnodes); MBERRNM(merr);
795: if (conn) *conn = connect;
796: if (nconn) *nconn = nnodes;
797: return(0);
798: }
801: /*@C
802: DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)
804: Collective
806: Input Parameter:
807: + dm - The DMMoab object
808: - ent - Entity handle
810: Output Parameter:
811: . ent_on_boundary - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise
813: Level: beginner
815: .seealso: DMMoabCheckBoundaryVertices()
816: @*/
817: PetscErrorCode DMMoabIsEntityOnBoundary(DM dm, const moab::EntityHandle ent, PetscBool* ent_on_boundary)
818: {
819: moab::EntityType etype;
820: DM_Moab *dmmoab;
821: PetscInt edim;
826: dmmoab = (DM_Moab*)(dm)->data;
828: /* get the entity type and handle accordingly */
829: etype = dmmoab->mbiface->type_from_handle(ent);
830: if (etype >= moab::MBPOLYHEDRON) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_OUTOFRANGE, "Entity type on the boundary skin is invalid. EntityType = %D\n", etype);
832: /* get the entity dimension */
833: edim = dmmoab->mbiface->dimension_from_handle(ent);
835: *ent_on_boundary = PETSC_FALSE;
836: if (etype == moab::MBVERTEX && edim == 0) {
837: *ent_on_boundary = ((dmmoab->bndyvtx->index(ent) >= 0) ? PETSC_TRUE : PETSC_FALSE);
838: }
839: else {
840: if (edim == dmmoab->dim) { /* check the higher-dimensional elements first */
841: if (dmmoab->bndyelems->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
842: }
843: else { /* next check the lower-dimensional faces */
844: if (dmmoab->bndyfaces->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
845: }
846: }
847: return(0);
848: }
851: /*@C
852: DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)
854: Input Parameter:
855: + dm - The DMMoab object
856: . nconn - Number of handles
857: - cnt - Array of entity handles
859: Output Parameter:
860: . isbdvtx - Array of boundary markers - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise
862: Level: beginner
864: .seealso: DMMoabIsEntityOnBoundary()
865: @*/
866: PetscErrorCode DMMoabCheckBoundaryVertices(DM dm, PetscInt nconn, const moab::EntityHandle *cnt, PetscBool* isbdvtx)
867: {
868: DM_Moab *dmmoab;
869: PetscInt i;
875: dmmoab = (DM_Moab*)(dm)->data;
877: for (i = 0; i < nconn; ++i) {
878: isbdvtx[i] = (dmmoab->bndyvtx->index(cnt[i]) >= 0 ? PETSC_TRUE : PETSC_FALSE);
879: }
880: return(0);
881: }
884: /*@C
885: DMMoabGetBoundaryMarkers - Return references to the vertices, faces, elements on the boundary
887: Input Parameter:
888: . dm - The DMMoab object
890: Output Parameter:
891: + bdvtx - Boundary vertices
892: . bdelems - Boundary elements
893: - bdfaces - Boundary faces
895: Level: beginner
897: .seealso: DMMoabCheckBoundaryVertices(), DMMoabIsEntityOnBoundary()
898: @*/
899: PetscErrorCode DMMoabGetBoundaryMarkers(DM dm, const moab::Range **bdvtx, const moab::Range** bdelems, const moab::Range** bdfaces)
900: {
901: DM_Moab *dmmoab;
905: dmmoab = (DM_Moab*)(dm)->data;
907: if (bdvtx) *bdvtx = dmmoab->bndyvtx;
908: if (bdfaces) *bdfaces = dmmoab->bndyfaces;
909: if (bdelems) *bdfaces = dmmoab->bndyelems;
910: return(0);
911: }
914: PETSC_EXTERN PetscErrorCode DMDestroy_Moab(DM dm)
915: {
916: PetscErrorCode ierr;
917: PetscInt i;
918: moab::ErrorCode merr;
919: DM_Moab *dmmoab = (DM_Moab*)dm->data;
924: dmmoab->refct--;
925: if (!dmmoab->refct) {
926: delete dmmoab->vlocal;
927: delete dmmoab->vowned;
928: delete dmmoab->vghost;
929: delete dmmoab->elocal;
930: delete dmmoab->eghost;
931: delete dmmoab->bndyvtx;
932: delete dmmoab->bndyfaces;
933: delete dmmoab->bndyelems;
935: PetscFree(dmmoab->gsindices);
936: PetscFree2(dmmoab->gidmap, dmmoab->lidmap);
937: PetscFree(dmmoab->dfill);
938: PetscFree(dmmoab->ofill);
939: PetscFree(dmmoab->materials);
940: if (dmmoab->fieldNames) {
941: for (i = 0; i < dmmoab->numFields; i++) {
942: PetscFree(dmmoab->fieldNames[i]);
943: }
944: PetscFree(dmmoab->fieldNames);
945: }
947: if (dmmoab->nhlevels) {
948: PetscFree(dmmoab->hsets);
949: dmmoab->nhlevels = 0;
950: if (!dmmoab->hlevel && dmmoab->icreatedinstance) delete dmmoab->hierarchy;
951: dmmoab->hierarchy = NULL;
952: }
954: if (dmmoab->icreatedinstance) {
955: delete dmmoab->pcomm;
956: merr = dmmoab->mbiface->delete_mesh(); MBERRNM(merr);
957: delete dmmoab->mbiface;
958: }
959: dmmoab->mbiface = NULL;
960: #ifdef MOAB_HAVE_MPI
961: dmmoab->pcomm = NULL;
962: #endif
963: VecScatterDestroy(&dmmoab->ltog_sendrecv);
964: ISLocalToGlobalMappingDestroy(&dmmoab->ltog_map);
965: PetscFree(dm->data);
966: }
967: return(0);
968: }
971: PETSC_EXTERN PetscErrorCode DMSetFromOptions_Moab(PetscOptionItems *PetscOptionsObject, DM dm)
972: {
974: DM_Moab *dmmoab = (DM_Moab*)dm->data;
978: PetscOptionsHead(PetscOptionsObject, "DMMoab Options");
979: PetscOptionsBoundedInt("-dm_moab_rw_dbg", "The verbosity level for reading and writing MOAB meshes", "DMView", dmmoab->rw_dbglevel, &dmmoab->rw_dbglevel, NULL,0);
980: PetscOptionsBool("-dm_moab_partiton_by_rank", "Use partition by rank when reading MOAB meshes from file", "DMView", dmmoab->partition_by_rank, &dmmoab->partition_by_rank, NULL);
981: /* TODO: typically, the read options are needed before a DM is completely created and available in which case, the options wont be available ?? */
982: PetscOptionsString("-dm_moab_read_opts", "Extra options to enable MOAB reader to load DM from file", "DMView", dmmoab->extra_read_options, dmmoab->extra_read_options, sizeof(dmmoab->extra_read_options), NULL);
983: PetscOptionsString("-dm_moab_write_opts", "Extra options to enable MOAB writer to serialize DM to file", "DMView", dmmoab->extra_write_options, dmmoab->extra_write_options, sizeof(dmmoab->extra_write_options), NULL);
984: PetscOptionsEnum("-dm_moab_read_mode", "MOAB parallel read mode", "DMView", MoabReadModes, (PetscEnum)dmmoab->read_mode, (PetscEnum*)&dmmoab->read_mode, NULL);
985: PetscOptionsEnum("-dm_moab_write_mode", "MOAB parallel write mode", "DMView", MoabWriteModes, (PetscEnum)dmmoab->write_mode, (PetscEnum*)&dmmoab->write_mode, NULL);
986: return(0);
987: }
990: PETSC_EXTERN PetscErrorCode DMSetUp_Moab(DM dm)
991: {
992: PetscErrorCode ierr;
993: moab::ErrorCode merr;
994: Vec local, global;
995: IS from, to;
996: moab::Range::iterator iter;
997: PetscInt i, j, f, bs, vent, totsize, *lgmap;
998: DM_Moab *dmmoab = (DM_Moab*)dm->data;
999: moab::Range adjs;
1003: /* Get the local and shared vertices and cache it */
1004: if (dmmoab->mbiface == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB Interface before calling SetUp.");
1005: #ifdef MOAB_HAVE_MPI
1006: if (dmmoab->pcomm == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB ParallelComm object before calling SetUp.");
1007: #endif
1009: /* Get the entities recursively in the current part of the mesh, if user did not set the local vertices explicitly */
1010: if (dmmoab->vlocal->empty())
1011: {
1012: //merr = dmmoab->mbiface->get_entities_by_type(dmmoab->fileset,moab::MBVERTEX,*dmmoab->vlocal,true);MBERRNM(merr);
1013: merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, 0, *dmmoab->vlocal, false); MBERRNM(merr);
1015: #ifdef MOAB_HAVE_MPI
1016: /* filter based on parallel status */
1017: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);
1019: /* filter all the non-owned and shared entities out of the list */
1020: // *dmmoab->vghost = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
1021: adjs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
1022: merr = dmmoab->pcomm->filter_pstatus(adjs, PSTATUS_GHOST | PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
1023: adjs = moab::subtract(adjs, *dmmoab->vghost);
1024: *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, adjs);
1025: #else
1026: *dmmoab->vowned = *dmmoab->vlocal;
1027: #endif
1029: /* compute and cache the sizes of local and ghosted entities */
1030: dmmoab->nloc = dmmoab->vowned->size();
1031: dmmoab->nghost = dmmoab->vghost->size();
1033: #ifdef MOAB_HAVE_MPI
1034: MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
1035: PetscInfo4(NULL, "Filset ID: %u, Vertices: local - %D, owned - %D, ghosted - %D.\n", dmmoab->fileset, dmmoab->vlocal->size(), dmmoab->nloc, dmmoab->nghost);
1036: #else
1037: dmmoab->n = dmmoab->nloc;
1038: #endif
1039: }
1041: {
1042: /* get the information about the local elements in the mesh */
1043: dmmoab->eghost->clear();
1045: /* first decipher the leading dimension */
1046: for (i = 3; i > 0; i--) {
1047: dmmoab->elocal->clear();
1048: merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, i, *dmmoab->elocal, false); MBERRNM(merr);
1050: /* store the current mesh dimension */
1051: if (dmmoab->elocal->size()) {
1052: dmmoab->dim = i;
1053: break;
1054: }
1055: }
1057: DMSetDimension(dm, dmmoab->dim);
1059: #ifdef MOAB_HAVE_MPI
1060: /* filter the ghosted and owned element list */
1061: *dmmoab->eghost = *dmmoab->elocal;
1062: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1063: *dmmoab->eghost = moab::subtract(*dmmoab->eghost, *dmmoab->elocal);
1064: #endif
1066: dmmoab->neleloc = dmmoab->elocal->size();
1067: dmmoab->neleghost = dmmoab->eghost->size();
1069: #ifdef MOAB_HAVE_MPI
1070: MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
1071: PetscInfo3(NULL, "%d-dim elements: owned - %D, ghosted - %D.\n", dmmoab->dim, dmmoab->neleloc, dmmoab->neleghost);
1072: #else
1073: dmmoab->nele = dmmoab->neleloc;
1074: #endif
1075: }
1077: bs = dmmoab->bs;
1078: if (!dmmoab->ltog_tag) {
1079: /* Get the global ID tag. The global ID tag is applied to each
1080: vertex. It acts as an global identifier which MOAB uses to
1081: assemble the individual pieces of the mesh */
1082: merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
1083: }
1085: totsize = dmmoab->vlocal->size();
1086: if (totsize != dmmoab->nloc + dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Mismatch between local and owned+ghost vertices. %D != %D.", totsize, dmmoab->nloc + dmmoab->nghost);
1087: PetscCalloc1(totsize, &dmmoab->gsindices);
1088: {
1089: /* first get the local indices */
1090: merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vowned, &dmmoab->gsindices[0]); MBERRNM(merr);
1091: if (dmmoab->nghost) { /* next get the ghosted indices */
1092: merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vghost, &dmmoab->gsindices[dmmoab->nloc]); MBERRNM(merr);
1093: }
1095: /* find out the local and global minima of GLOBAL_ID */
1096: dmmoab->lminmax[0] = dmmoab->lminmax[1] = dmmoab->gsindices[0];
1097: for (i = 0; i < totsize; ++i) {
1098: if (dmmoab->lminmax[0] > dmmoab->gsindices[i]) dmmoab->lminmax[0] = dmmoab->gsindices[i];
1099: if (dmmoab->lminmax[1] < dmmoab->gsindices[i]) dmmoab->lminmax[1] = dmmoab->gsindices[i];
1100: }
1102: MPIU_Allreduce(&dmmoab->lminmax[0], &dmmoab->gminmax[0], 1, MPI_INT, MPI_MIN, ((PetscObject)dm)->comm);
1103: MPIU_Allreduce(&dmmoab->lminmax[1], &dmmoab->gminmax[1], 1, MPI_INT, MPI_MAX, ((PetscObject)dm)->comm);
1105: /* set the GID map */
1106: for (i = 0; i < totsize; ++i) {
1107: dmmoab->gsindices[i] -= dmmoab->gminmax[0]; /* zero based index needed for IS */
1109: }
1110: dmmoab->lminmax[0] -= dmmoab->gminmax[0];
1111: dmmoab->lminmax[1] -= dmmoab->gminmax[0];
1113: PetscInfo4(NULL, "GLOBAL_ID: Local [min, max] - [%D, %D], Global [min, max] - [%D, %D]\n", dmmoab->lminmax[0], dmmoab->lminmax[1], dmmoab->gminmax[0], dmmoab->gminmax[1]);
1114: }
1115: if (!(dmmoab->bs == dmmoab->numFields || dmmoab->bs == 1)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between block size and number of component fields. %D != 1 OR %D != %D.", dmmoab->bs, dmmoab->bs, dmmoab->numFields);
1117: {
1118: dmmoab->seqstart = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->front());
1119: dmmoab->seqend = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->back());
1120: PetscInfo2(NULL, "SEQUENCE: Local [min, max] - [%D, %D]\n", dmmoab->seqstart, dmmoab->seqend);
1122: PetscMalloc2(dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->gidmap, dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->lidmap);
1123: PetscMalloc1(totsize * dmmoab->numFields, &lgmap);
1125: i = j = 0;
1126: /* set the owned vertex data first */
1127: for (moab::Range::iterator iter = dmmoab->vowned->begin(); iter != dmmoab->vowned->end(); iter++, i++) {
1128: vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
1129: dmmoab->gidmap[vent] = dmmoab->gsindices[i];
1130: dmmoab->lidmap[vent] = i;
1131: for (f = 0; f < dmmoab->numFields; f++, j++) {
1132: lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
1133: }
1134: }
1135: /* next arrange all the ghosted data information */
1136: for (moab::Range::iterator iter = dmmoab->vghost->begin(); iter != dmmoab->vghost->end(); iter++, i++) {
1137: vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
1138: dmmoab->gidmap[vent] = dmmoab->gsindices[i];
1139: dmmoab->lidmap[vent] = i;
1140: for (f = 0; f < dmmoab->numFields; f++, j++) {
1141: lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
1142: }
1143: }
1145: /* We need to create the Global to Local Vector Scatter Contexts
1146: 1) First create a local and global vector
1147: 2) Create a local and global IS
1148: 3) Create VecScatter and LtoGMapping objects
1149: 4) Cleanup the IS and Vec objects
1150: */
1151: DMCreateGlobalVector(dm, &global);
1152: DMCreateLocalVector(dm, &local);
1154: VecGetOwnershipRange(global, &dmmoab->vstart, &dmmoab->vend);
1156: /* global to local must retrieve ghost points */
1157: ISCreateStride(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, dmmoab->vstart, 1, &from);
1158: ISSetBlockSize(from, bs);
1160: ISCreateGeneral(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, &lgmap[0], PETSC_COPY_VALUES, &to);
1161: ISSetBlockSize(to, bs);
1163: if (!dmmoab->ltog_map) {
1164: /* create to the local to global mapping for vectors in order to use VecSetValuesLocal */
1165: ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm, dmmoab->bs, totsize * dmmoab->numFields, lgmap,
1166: PETSC_COPY_VALUES, &dmmoab->ltog_map);
1167: }
1169: /* now create the scatter object from local to global vector */
1170: VecScatterCreate(local, from, global, to, &dmmoab->ltog_sendrecv);
1172: /* clean up IS, Vec */
1173: PetscFree(lgmap);
1174: ISDestroy(&from);
1175: ISDestroy(&to);
1176: VecDestroy(&local);
1177: VecDestroy(&global);
1178: }
1180: dmmoab->bndyvtx = new moab::Range();
1181: dmmoab->bndyfaces = new moab::Range();
1182: dmmoab->bndyelems = new moab::Range();
1183: /* skin the boundary and store nodes */
1184: if (!dmmoab->hlevel) {
1185: /* get the skin vertices of boundary faces for the current partition and then filter
1186: the local, boundary faces, vertices and elements alone via PSTATUS flags;
1187: this should not give us any ghosted boundary, but if user needs such a functionality
1188: it would be easy to add it based on the find_skin query below */
1189: moab::Skinner skinner(dmmoab->mbiface);
1191: /* get the entities on the skin - only the faces */
1192: merr = skinner.find_skin(dmmoab->fileset, *dmmoab->elocal, false, *dmmoab->bndyfaces, NULL, true, true, false); MBERRNM(merr); // 'false' param indicates we want faces back, not vertices
1194: #ifdef MOAB_HAVE_MPI
1195: /* filter all the non-owned and shared entities out of the list */
1196: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1197: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_INTERFACE, PSTATUS_NOT); MBERRNM(merr);
1198: #endif
1200: /* get all the nodes via connectivity and the parent elements via adjacency information */
1201: merr = dmmoab->mbiface->get_connectivity(*dmmoab->bndyfaces, *dmmoab->bndyvtx, false); MBERRNM(ierr);
1202: merr = dmmoab->mbiface->get_adjacencies(*dmmoab->bndyvtx, dmmoab->dim, false, *dmmoab->bndyelems, moab::Interface::UNION); MBERRNM(ierr);
1203: }
1204: else {
1205: /* Let us query the hierarchy manager and get the results directly for this level */
1206: for (moab::Range::iterator iter = dmmoab->elocal->begin(); iter != dmmoab->elocal->end(); iter++) {
1207: moab::EntityHandle elemHandle = *iter;
1208: if (dmmoab->hierarchy->is_entity_on_boundary(elemHandle)) {
1209: dmmoab->bndyelems->insert(elemHandle);
1210: /* For this boundary element, query the vertices and add them to the list */
1211: std::vector<moab::EntityHandle> connect;
1212: merr = dmmoab->hierarchy->get_connectivity(elemHandle, dmmoab->hlevel, connect); MBERRNM(ierr);
1213: for (unsigned iv=0; iv < connect.size(); ++iv)
1214: if (dmmoab->hierarchy->is_entity_on_boundary(connect[iv]))
1215: dmmoab->bndyvtx->insert(connect[iv]);
1216: /* Next, let us query the boundary faces and add them also to the list */
1217: std::vector<moab::EntityHandle> faces;
1218: merr = dmmoab->hierarchy->get_adjacencies(elemHandle, dmmoab->dim-1, faces); MBERRNM(ierr);
1219: for (unsigned ifa=0; ifa < faces.size(); ++ifa)
1220: if (dmmoab->hierarchy->is_entity_on_boundary(faces[ifa]))
1221: dmmoab->bndyfaces->insert(faces[ifa]);
1222: }
1223: }
1224: #ifdef MOAB_HAVE_MPI
1225: /* filter all the non-owned and shared entities out of the list */
1226: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyvtx, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1227: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1228: merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1229: #endif
1231: }
1232: PetscInfo3(NULL, "Found %D boundary vertices, %D boundary faces and %D boundary elements.\n", dmmoab->bndyvtx->size(), dmmoab->bndyfaces->size(), dmmoab->bndyelems->size());
1234: /* Get the material sets and populate the data for all locally owned elements */
1235: {
1236: PetscCalloc1(dmmoab->elocal->size(), &dmmoab->materials);
1237: /* Get the count of entities of particular type from dmmoab->elocal
1238: -- Then, for each non-zero type, loop through and query the fileset to get the material tag data */
1239: moab::Range msets;
1240: merr = dmmoab->mbiface->get_entities_by_type_and_tag(dmmoab->fileset, moab::MBENTITYSET, &dmmoab->material_tag, NULL, 1, msets, moab::Interface::UNION);MB_CHK_ERR(merr);
1241: if (msets.size() == 0) {
1242: PetscInfo(NULL, "No material sets found in the fileset.");
1243: }
1245: for (unsigned i=0; i < msets.size(); ++i) {
1246: moab::Range msetelems;
1247: merr = dmmoab->mbiface->get_entities_by_dimension(msets[i], dmmoab->dim, msetelems, true);MB_CHK_ERR(merr);
1248: #ifdef MOAB_HAVE_MPI
1249: /* filter all the non-owned and shared entities out of the list */
1250: merr = dmmoab->pcomm->filter_pstatus(msetelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
1251: #endif
1253: int partID;
1254: moab::EntityHandle mset=msets[i];
1255: merr = dmmoab->mbiface->tag_get_data(dmmoab->material_tag, &mset, 1, &partID);MB_CHK_ERR(merr);
1257: for (unsigned j=0; j < msetelems.size(); ++j)
1258: dmmoab->materials[dmmoab->elocal->index(msetelems[j])]=partID;
1259: }
1260: }
1262: return(0);
1263: }
1266: /*@C
1267: DMMoabCreateVertices - Creates and adds several vertices to the primary set represented by the DM.
1269: Collective
1271: Input Parameters:
1272: + dm - The DM object
1273: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
1274: . conn - The connectivity of the element
1275: - nverts - The number of vertices that form the element
1277: Output Parameter:
1278: . overts - The list of vertices that were created (can be NULL)
1280: Level: beginner
1282: .seealso: DMMoabCreateSubmesh(), DMMoabCreateElement()
1283: @*/
1284: PetscErrorCode DMMoabCreateVertices(DM dm, const PetscReal* coords, PetscInt nverts, moab::Range* overts)
1285: {
1286: moab::ErrorCode merr;
1287: DM_Moab *dmmoab;
1288: moab::Range verts;
1294: dmmoab = (DM_Moab*) dm->data;
1296: /* Insert new points */
1297: merr = dmmoab->mbiface->create_vertices(&coords[0], nverts, verts); MBERRNM(merr);
1298: merr = dmmoab->mbiface->add_entities(dmmoab->fileset, verts); MBERRNM(merr);
1300: if (overts) *overts = verts;
1301: return(0);
1302: }
1305: /*@C
1306: DMMoabCreateElement - Adds an element of specified type to the primary set represented by the DM.
1308: Collective
1310: Input Parameters:
1311: + dm - The DM object
1312: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
1313: . conn - The connectivity of the element
1314: - nverts - The number of vertices that form the element
1316: Output Parameter:
1317: . oelem - The handle to the element created and added to the DM object
1319: Level: beginner
1321: .seealso: DMMoabCreateSubmesh(), DMMoabCreateVertices()
1322: @*/
1323: PetscErrorCode DMMoabCreateElement(DM dm, const moab::EntityType type, const moab::EntityHandle* conn, PetscInt nverts, moab::EntityHandle* oelem)
1324: {
1325: moab::ErrorCode merr;
1326: DM_Moab *dmmoab;
1327: moab::EntityHandle elem;
1333: dmmoab = (DM_Moab*) dm->data;
1335: /* Insert new element */
1336: merr = dmmoab->mbiface->create_element(type, conn, nverts, elem); MBERRNM(merr);
1337: merr = dmmoab->mbiface->add_entities(dmmoab->fileset, &elem, 1); MBERRNM(merr);
1339: if (oelem) *oelem = elem;
1340: return(0);
1341: }
1344: /*@C
1345: DMMoabCreateSubmesh - Creates a sub-DM object with a set that contains all vertices/elements of the parent
1346: in addition to providing support for dynamic mesh modifications. This is useful for AMR calculations to
1347: create a DM object on a refined level.
1349: Collective
1351: Input Parameters:
1352: . dm - The DM object
1354: Output Parameter:
1355: . newdm - The sub DM object with updated set information
1357: Level: advanced
1359: .seealso: DMCreate(), DMMoabCreateVertices(), DMMoabCreateElement()
1360: @*/
1361: PetscErrorCode DMMoabCreateSubmesh(DM dm, DM *newdm)
1362: {
1363: DM_Moab *dmmoab;
1364: DM_Moab *ndmmoab;
1365: moab::ErrorCode merr;
1366: PetscErrorCode ierr;
1371: dmmoab = (DM_Moab*) dm->data;
1373: /* Create the basic DMMoab object and keep the default parameters created by DM impls */
1374: DMMoabCreateMoab(((PetscObject)dm)->comm, dmmoab->mbiface, &dmmoab->ltog_tag, PETSC_NULL, newdm);
1376: /* get all the necessary handles from the private DM object */
1377: ndmmoab = (DM_Moab*) (*newdm)->data;
1379: /* set the sub-mesh's parent DM reference */
1380: ndmmoab->parent = &dm;
1382: /* create a file set to associate all entities in current mesh */
1383: merr = ndmmoab->mbiface->create_meshset(moab::MESHSET_SET, ndmmoab->fileset); MBERR("Creating file set failed", merr);
1385: /* create a meshset and then add old fileset as child */
1386: merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->vlocal); MBERR("Adding child vertices to parent failed", merr);
1387: merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->elocal); MBERR("Adding child elements to parent failed", merr);
1389: /* preserve the field association between the parent and sub-mesh objects */
1390: DMMoabSetFieldNames(*newdm, dmmoab->numFields, dmmoab->fieldNames);
1391: return(0);
1392: }
1395: PETSC_EXTERN PetscErrorCode DMMoabView_Ascii(DM dm, PetscViewer viewer)
1396: {
1397: DM_Moab *dmmoab = (DM_Moab*)(dm)->data;
1398: const char *name;
1399: MPI_Comm comm;
1400: PetscMPIInt size;
1401: PetscErrorCode ierr;
1404: PetscObjectGetComm((PetscObject)dm, &comm);
1405: MPI_Comm_size(comm, &size);
1406: PetscObjectGetName((PetscObject) dm, &name);
1407: PetscViewerASCIIPushTab(viewer);
1408: if (name) {PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dmmoab->dim);}
1409: else {PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dmmoab->dim);}
1410: /* print details about the global mesh */
1411: {
1412: PetscViewerASCIIPushTab(viewer);
1413: PetscViewerASCIIPrintf(viewer, "Sizes: cells=%D, vertices=%D, blocks=%D\n", dmmoab->nele, dmmoab->n, dmmoab->bs);
1414: /* print boundary data */
1415: PetscViewerASCIIPrintf(viewer, "Boundary trace:\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
1416: {
1417: PetscViewerASCIIPushTab(viewer);
1418: PetscViewerASCIIPrintf(viewer, "cells=%D, faces=%D, vertices=%D\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
1419: PetscViewerASCIIPopTab(viewer);
1420: }
1421: /* print field data */
1422: PetscViewerASCIIPrintf(viewer, "Fields: %D components\n", dmmoab->numFields);
1423: {
1424: PetscViewerASCIIPushTab(viewer);
1425: for (int i = 0; i < dmmoab->numFields; ++i) {
1426: PetscViewerASCIIPrintf(viewer, "[%D] - %s\n", i, dmmoab->fieldNames[i]);
1427: }
1428: PetscViewerASCIIPopTab(viewer);
1429: }
1430: PetscViewerASCIIPopTab(viewer);
1431: }
1432: PetscViewerASCIIPopTab(viewer);
1433: PetscViewerFlush(viewer);
1434: return(0);
1435: }
1437: PETSC_EXTERN PetscErrorCode DMMoabView_VTK(DM dm, PetscViewer v)
1438: {
1439: return(0);
1440: }
1442: PETSC_EXTERN PetscErrorCode DMMoabView_HDF5(DM dm, PetscViewer v)
1443: {
1444: return(0);
1445: }
1447: PETSC_EXTERN PetscErrorCode DMView_Moab(DM dm, PetscViewer viewer)
1448: {
1449: PetscBool iascii, ishdf5, isvtk;
1455: PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);
1456: PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);
1457: PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);
1458: if (iascii) {
1459: DMMoabView_Ascii(dm, viewer);
1460: } else if (ishdf5) {
1461: #if defined(PETSC_HAVE_HDF5) && defined(MOAB_HAVE_HDF5)
1462: PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);
1463: DMMoabView_HDF5(dm, viewer);
1464: PetscViewerPopFormat(viewer);
1465: #else
1466: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1467: #endif
1468: }
1469: else if (isvtk) {
1470: DMMoabView_VTK(dm, viewer);
1471: }
1472: return(0);
1473: }
1476: PETSC_EXTERN PetscErrorCode DMInitialize_Moab(DM dm)
1477: {
1479: dm->ops->view = DMView_Moab;
1480: dm->ops->load = NULL /* DMLoad_Moab */;
1481: dm->ops->setfromoptions = DMSetFromOptions_Moab;
1482: dm->ops->clone = DMClone_Moab;
1483: dm->ops->setup = DMSetUp_Moab;
1484: dm->ops->createlocalsection = NULL;
1485: dm->ops->createdefaultconstraints = NULL;
1486: dm->ops->createglobalvector = DMCreateGlobalVector_Moab;
1487: dm->ops->createlocalvector = DMCreateLocalVector_Moab;
1488: dm->ops->getlocaltoglobalmapping = NULL;
1489: dm->ops->createfieldis = NULL;
1490: dm->ops->createcoordinatedm = NULL /* DMCreateCoordinateDM_Moab */;
1491: dm->ops->getcoloring = NULL;
1492: dm->ops->creatematrix = DMCreateMatrix_Moab;
1493: dm->ops->createinterpolation = DMCreateInterpolation_Moab;
1494: dm->ops->createinjection = NULL /* DMCreateInjection_Moab */;
1495: dm->ops->refine = DMRefine_Moab;
1496: dm->ops->coarsen = DMCoarsen_Moab;
1497: dm->ops->refinehierarchy = DMRefineHierarchy_Moab;
1498: dm->ops->coarsenhierarchy = DMCoarsenHierarchy_Moab;
1499: dm->ops->globaltolocalbegin = DMGlobalToLocalBegin_Moab;
1500: dm->ops->globaltolocalend = DMGlobalToLocalEnd_Moab;
1501: dm->ops->localtoglobalbegin = DMLocalToGlobalBegin_Moab;
1502: dm->ops->localtoglobalend = DMLocalToGlobalEnd_Moab;
1503: dm->ops->destroy = DMDestroy_Moab;
1504: dm->ops->createsubdm = NULL /* DMCreateSubDM_Moab */;
1505: dm->ops->getdimpoints = NULL /* DMGetDimPoints_Moab */;
1506: dm->ops->locatepoints = NULL /* DMLocatePoints_Moab */;
1507: return(0);
1508: }
1511: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm)
1512: {
1513: PetscErrorCode ierr;
1516: /* get all the necessary handles from the private DM object */
1517: (*newdm)->data = (DM_Moab*) dm->data;
1518: ((DM_Moab*)dm->data)->refct++;
1520: PetscObjectChangeTypeName((PetscObject) *newdm, DMMOAB);
1521: DMInitialize_Moab(*newdm);
1522: return(0);
1523: }
1526: PETSC_EXTERN PetscErrorCode DMCreate_Moab(DM dm)
1527: {
1532: PetscNewLog(dm, (DM_Moab**)&dm->data);
1534: ((DM_Moab*)dm->data)->bs = 1;
1535: ((DM_Moab*)dm->data)->numFields = 1;
1536: ((DM_Moab*)dm->data)->n = 0;
1537: ((DM_Moab*)dm->data)->nloc = 0;
1538: ((DM_Moab*)dm->data)->nghost = 0;
1539: ((DM_Moab*)dm->data)->nele = 0;
1540: ((DM_Moab*)dm->data)->neleloc = 0;
1541: ((DM_Moab*)dm->data)->neleghost = 0;
1542: ((DM_Moab*)dm->data)->ltog_map = NULL;
1543: ((DM_Moab*)dm->data)->ltog_sendrecv = NULL;
1545: ((DM_Moab*)dm->data)->refct = 1;
1546: ((DM_Moab*)dm->data)->parent = NULL;
1547: ((DM_Moab*)dm->data)->vlocal = new moab::Range();
1548: ((DM_Moab*)dm->data)->vowned = new moab::Range();
1549: ((DM_Moab*)dm->data)->vghost = new moab::Range();
1550: ((DM_Moab*)dm->data)->elocal = new moab::Range();
1551: ((DM_Moab*)dm->data)->eghost = new moab::Range();
1553: DMInitialize_Moab(dm);
1554: return(0);
1555: }