Actual source code: plexhdf5.c

  1: #include <petsc/private/dmpleximpl.h>
  2: #include <petsc/private/isimpl.h>
  3: #include <petsc/private/vecimpl.h>
  4: #include <petsc/private/viewerhdf5impl.h>
  5: #include <petsclayouthdf5.h>

  7: /* Logging support */
  8: PetscLogEvent DMPLEX_DistributionView, DMPLEX_DistributionLoad;

 10: #if defined(PETSC_HAVE_HDF5)
 11: static PetscErrorCode PetscViewerParseVersion_Private(PetscViewer, const char[], DMPlexStorageVersion *);
 12: static PetscErrorCode PetscViewerCheckVersion_Private(PetscViewer, DMPlexStorageVersion);
 13: static PetscErrorCode PetscViewerAttachVersion_Private(PetscViewer, const char[], DMPlexStorageVersion);
 14: static PetscErrorCode PetscViewerGetAttachedVersion_Private(PetscViewer, const char[], DMPlexStorageVersion *);

 16: PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);

 18: static PetscErrorCode PetscViewerPrintVersion_Private(PetscViewer viewer, DMPlexStorageVersion version, char str[], size_t len)
 19: {
 20:   PetscFunctionBegin;
 21:   PetscCall(PetscViewerCheckVersion_Private(viewer, version));
 22:   PetscCall(PetscSNPrintf(str, len, "%d.%d.%d", version->major, version->minor, version->subminor));
 23:   PetscFunctionReturn(PETSC_SUCCESS);
 24: }

 26: static PetscErrorCode PetscViewerParseVersion_Private(PetscViewer viewer, const char str[], DMPlexStorageVersion *version)
 27: {
 28:   PetscToken           t;
 29:   char                *ts;
 30:   PetscInt             i;
 31:   PetscInt             ti[3];
 32:   DMPlexStorageVersion v;

 34:   PetscFunctionBegin;
 35:   PetscCall(PetscTokenCreate(str, '.', &t));
 36:   for (i = 0; i < 3; i++) {
 37:     PetscCall(PetscTokenFind(t, &ts));
 38:     PetscCheck(ts, PetscObjectComm((PetscObject)viewer), PETSC_ERR_ARG_WRONG, "Malformed version string %s", str);
 39:     PetscCall(PetscOptionsStringToInt(ts, &ti[i]));
 40:   }
 41:   PetscCall(PetscTokenFind(t, &ts));
 42:   PetscCheck(!ts, PetscObjectComm((PetscObject)viewer), PETSC_ERR_ARG_WRONG, "Malformed version string %s", str);
 43:   PetscCall(PetscTokenDestroy(&t));
 44:   PetscCall(PetscNew(&v));
 45:   v->major    = (int)ti[0];
 46:   v->minor    = (int)ti[1];
 47:   v->subminor = (int)ti[2];
 48:   PetscCall(PetscViewerCheckVersion_Private(viewer, v));
 49:   *version = v;
 50:   PetscFunctionReturn(PETSC_SUCCESS);
 51: }

 53: static PetscErrorCode PetscViewerAttachVersion_Private(PetscViewer viewer, const char key[], DMPlexStorageVersion v)
 54: {
 55:   PetscFunctionBegin;
 56:   PetscCall(PetscObjectContainerCompose((PetscObject)viewer, key, v, PetscContainerUserDestroyDefault));
 57:   PetscFunctionReturn(PETSC_SUCCESS);
 58: }

 60: static PetscErrorCode PetscViewerGetAttachedVersion_Private(PetscViewer viewer, const char key[], DMPlexStorageVersion *v)
 61: {
 62:   PetscContainer cont;

 64:   PetscFunctionBegin;
 65:   PetscCall(PetscObjectQuery((PetscObject)viewer, key, (PetscObject *)&cont));
 66:   *v = NULL;
 67:   if (cont) PetscCall(PetscContainerGetPointer(cont, (void **)v));
 68:   PetscFunctionReturn(PETSC_SUCCESS);
 69: }

 71: /*
 72:   Version log:
 73:   1.0.0 legacy version (default if no "dmplex_storage_version" attribute found in file)
 74:   1.1.0 legacy version, but output VIZ by default
 75:   2.0.0 introduce versioning and multiple topologies
 76:   2.1.0 introduce distributions
 77:   3.0.0 new checkpointing format in Firedrake paper
 78:   3.1.0 new format with IS compression
 79: */
 80: static PetscErrorCode PetscViewerCheckVersion_Private(PetscViewer viewer, DMPlexStorageVersion version)
 81: {
 82:   PetscBool valid = PETSC_FALSE;

 84:   PetscFunctionBegin;
 85:   switch (version->major) {
 86:   case 1:
 87:     switch (version->minor) {
 88:     case 0:
 89:       switch (version->subminor) {
 90:       case 0:
 91:         valid = PETSC_TRUE;
 92:         break;
 93:       }
 94:       break;
 95:     case 1:
 96:       switch (version->subminor) {
 97:       case 0:
 98:         valid = PETSC_TRUE;
 99:         break;
100:       }
101:       break;
102:     }
103:     break;
104:   case 2:
105:     switch (version->minor) {
106:     case 0:
107:       switch (version->subminor) {
108:       case 0:
109:         valid = PETSC_TRUE;
110:         break;
111:       }
112:       break;
113:     case 1:
114:       switch (version->subminor) {
115:       case 0:
116:         valid = PETSC_TRUE;
117:         break;
118:       }
119:       break;
120:     }
121:     break;
122:   case 3:
123:     switch (version->minor) {
124:     case 0:
125:       switch (version->subminor) {
126:       case 0:
127:         valid = PETSC_TRUE;
128:         break;
129:       }
130:       break;
131:     case 1:
132:       switch (version->subminor) {
133:       case 0:
134:         valid = PETSC_TRUE;
135:         break;
136:       }
137:       break;
138:     }
139:     break;
140:   }
141:   PetscCheck(valid, PetscObjectComm((PetscObject)viewer), PETSC_ERR_SUP, "DMPlexStorageVersion %d.%d.%d not supported", version->major, version->minor, version->subminor);
142:   PetscFunctionReturn(PETSC_SUCCESS);
143: }

145: static inline PetscBool DMPlexStorageVersionEQ(DMPlexStorageVersion version, int major, int minor, int subminor)
146: {
147:   return (PetscBool)(version->major == major && version->minor == minor && version->subminor == subminor);
148: }

150: static inline PetscBool DMPlexStorageVersionGE(DMPlexStorageVersion version, int major, int minor, int subminor)
151: {
152:   return (PetscBool)((version->major == major && version->minor == minor && version->subminor >= subminor) || (version->major == major && version->minor > minor) || (version->major > major));
153: }

155: /*@C
156:   PetscViewerHDF5SetDMPlexStorageVersionWriting - Set the storage version for writing

158:   Logically collective

160:   Input Parameters:
161: + viewer  - The `PetscViewer`
162: - version - The storage format version

164:   Level: advanced

166:   Note:
167:   The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.

169: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`
170: @*/
171: PetscErrorCode PetscViewerHDF5SetDMPlexStorageVersionWriting(PetscViewer viewer, DMPlexStorageVersion version)
172: {
173:   const char           ATTR_NAME[] = "dmplex_storage_version";
174:   DMPlexStorageVersion viewerVersion;
175:   PetscBool            fileHasVersion;
176:   char                 fileVersion[16], versionStr[16], viewerVersionStr[16];

178:   PetscFunctionBegin;
180:   PetscAssertPointer(version, 2);
181:   PetscCall(PetscViewerPrintVersion_Private(viewer, version, versionStr, 16));
182:   PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, &viewerVersion));
183:   if (viewerVersion) {
184:     PetscBool flg;

186:     PetscCall(PetscViewerPrintVersion_Private(viewer, viewerVersion, viewerVersionStr, 16));
187:     PetscCall(PetscStrcmp(versionStr, viewerVersionStr, &flg));
188:     PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but viewer already has version %s - cannot mix versions", versionStr, viewerVersionStr);
189:   }

191:   PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
192:   if (fileHasVersion) {
193:     PetscBool flg;
194:     char     *tmp;

196:     PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &tmp));
197:     PetscCall(PetscStrncpy(fileVersion, tmp, sizeof(fileVersion)));
198:     PetscCall(PetscFree(tmp));
199:     PetscCall(PetscStrcmp(fileVersion, versionStr, &flg));
200:     PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", versionStr, fileVersion);
201:   } else {
202:     PetscCall(PetscViewerHDF5WriteAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, versionStr));
203:   }
204:   PetscCall(PetscNew(&viewerVersion));
205:   viewerVersion->major    = version->major;
206:   viewerVersion->minor    = version->minor;
207:   viewerVersion->subminor = version->subminor;
208:   PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, viewerVersion));
209:   PetscFunctionReturn(PETSC_SUCCESS);
210: }

212: /*@C
213:   PetscViewerHDF5GetDMPlexStorageVersionWriting - Get the storage version for writing

215:   Logically collective

217:   Input Parameter:
218: . viewer - The `PetscViewer`

220:   Output Parameter:
221: . version - The storage format version

223:   Options Database Keys:
224: . -dm_plex_view_hdf5_storage_version <num> - Overrides the storage format version

226:   Level: advanced

228:   Note:
229:   The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.

231: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`
232: @*/
233: PetscErrorCode PetscViewerHDF5GetDMPlexStorageVersionWriting(PetscViewer viewer, DMPlexStorageVersion *version)
234: {
235:   const char ATTR_NAME[] = "dmplex_storage_version";
236:   PetscBool  fileHasVersion;
237:   char       optVersion[16], fileVersion[16];

239:   PetscFunctionBegin;
241:   PetscAssertPointer(version, 2);
242:   PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, version));
243:   if (*version) PetscFunctionReturn(PETSC_SUCCESS);

245:   PetscCall(PetscStrncpy(fileVersion, DMPLEX_STORAGE_VERSION_STABLE, sizeof(fileVersion)));
246:   PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
247:   if (fileHasVersion) {
248:     char *tmp;

250:     PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &tmp));
251:     PetscCall(PetscStrncpy(fileVersion, tmp, sizeof(fileVersion)));
252:     PetscCall(PetscFree(tmp));
253:   }
254:   PetscCall(PetscStrncpy(optVersion, fileVersion, sizeof(optVersion)));
255:   PetscObjectOptionsBegin((PetscObject)viewer);
256:   PetscCall(PetscOptionsString("-dm_plex_view_hdf5_storage_version", "DMPlex HDF5 viewer storage version", NULL, optVersion, optVersion, sizeof(optVersion), NULL));
257:   PetscOptionsEnd();
258:   if (!fileHasVersion) {
259:     PetscCall(PetscViewerHDF5WriteAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, optVersion));
260:   } else {
261:     PetscBool flg;

263:     PetscCall(PetscStrcmp(fileVersion, optVersion, &flg));
264:     PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", optVersion, fileVersion);
265:   }
266:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "petsc_version_git", PETSC_STRING, PETSC_VERSION_GIT));
267:   PetscCall(PetscViewerParseVersion_Private(viewer, optVersion, version));
268:   PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, *version));
269:   PetscFunctionReturn(PETSC_SUCCESS);
270: }

272: /*@C
273:   PetscViewerHDF5SetDMPlexStorageVersionReading - Set the storage version for reading

275:   Logically collective

277:   Input Parameters:
278: + viewer  - The `PetscViewer`
279: - version - The storage format version

281:   Level: advanced

283:   Note:
284:   The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.

286: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`
287: @*/
288: PetscErrorCode PetscViewerHDF5SetDMPlexStorageVersionReading(PetscViewer viewer, DMPlexStorageVersion version)
289: {
290:   const char           ATTR_NAME[] = "dmplex_storage_version";
291:   DMPlexStorageVersion viewerVersion;
292:   PetscBool            fileHasVersion;
293:   char                 versionStr[16], viewerVersionStr[16];

295:   PetscFunctionBegin;
297:   PetscAssertPointer(version, 2);
298:   PetscCall(PetscViewerPrintVersion_Private(viewer, version, versionStr, 16));
299:   PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, &viewerVersion));
300:   if (viewerVersion) {
301:     PetscBool flg;

303:     PetscCall(PetscViewerPrintVersion_Private(viewer, viewerVersion, viewerVersionStr, 16));
304:     PetscCall(PetscStrcmp(versionStr, viewerVersionStr, &flg));
305:     PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but viewer already has version %s - cannot mix versions", versionStr, viewerVersionStr);
306:   }

308:   PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
309:   if (fileHasVersion) {
310:     char     *fileVersion;
311:     PetscBool flg;

313:     PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &fileVersion));
314:     PetscCall(PetscStrcmp(fileVersion, versionStr, &flg));
315:     PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", versionStr, fileVersion);
316:     PetscCall(PetscFree(fileVersion));
317:   }
318:   PetscCall(PetscNew(&viewerVersion));
319:   viewerVersion->major    = version->major;
320:   viewerVersion->minor    = version->minor;
321:   viewerVersion->subminor = version->subminor;
322:   PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, viewerVersion));
323:   PetscFunctionReturn(PETSC_SUCCESS);
324: }

326: /*@C
327:   PetscViewerHDF5GetDMPlexStorageVersionReading - Get the storage version for reading

329:   Logically collective

331:   Input Parameter:
332: . viewer - The `PetscViewer`

334:   Output Parameter:
335: . version - The storage format version

337:   Options Database Keys:
338: . -dm_plex_view_hdf5_storage_version <num> - Overrides the storage format version

340:   Level: advanced

342:   Note:
343:   The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.

345: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`
346: @*/
347: PetscErrorCode PetscViewerHDF5GetDMPlexStorageVersionReading(PetscViewer viewer, DMPlexStorageVersion *version)
348: {
349:   const char ATTR_NAME[] = "dmplex_storage_version";
350:   char      *defaultVersion;
351:   char      *versionString;

353:   PetscFunctionBegin;
355:   PetscAssertPointer(version, 2);
356:   PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, version));
357:   if (*version) PetscFunctionReturn(PETSC_SUCCESS);

359:   //TODO string HDF5 attribute handling is terrible and should be redesigned
360:   PetscCall(PetscStrallocpy(DMPLEX_STORAGE_VERSION_FIRST, &defaultVersion));
361:   PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, &defaultVersion, &versionString));
362:   PetscCall(PetscViewerParseVersion_Private(viewer, versionString, version));
363:   PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, *version));
364:   PetscCall(PetscFree(versionString));
365:   PetscCall(PetscFree(defaultVersion));
366:   PetscFunctionReturn(PETSC_SUCCESS);
367: }

369: static PetscErrorCode DMPlexGetHDF5Name_Private(DM dm, const char *name[])
370: {
371:   PetscFunctionBegin;
372:   if (((PetscObject)dm)->name) {
373:     PetscCall(PetscObjectGetName((PetscObject)dm, name));
374:   } else {
375:     *name = "plex";
376:   }
377:   PetscFunctionReturn(PETSC_SUCCESS);
378: }

380: PetscErrorCode DMSequenceGetLength_HDF5_Internal(DM dm, const char seqname[], PetscInt *seqlen, PetscViewer viewer)
381: {
382:   hid_t     file, group, dset, dspace;
383:   hsize_t   rdim, *dims;
384:   char     *groupname;
385:   PetscBool has;

387:   PetscFunctionBegin;
388:   PetscCall(PetscViewerHDF5GetGroup(viewer, NULL, &groupname));
389:   PetscCall(PetscViewerHDF5HasDataset(viewer, seqname, &has));
390:   PetscCheck(has, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "Object (dataset) \"%s\" not stored in group %s", seqname, groupname);

392:   PetscCall(PetscViewerHDF5OpenGroup(viewer, NULL, &file, &group));
393:   PetscCallHDF5Return(dset, H5Dopen2, (group, seqname, H5P_DEFAULT));
394:   PetscCallHDF5Return(dspace, H5Dget_space, (dset));
395:   PetscCallHDF5ReturnNoCheck(rdim, H5Sget_simple_extent_dims, (dspace, NULL, NULL));
396:   PetscCall(PetscMalloc1(rdim, &dims));
397:   PetscCallHDF5ReturnNoCheck(rdim, H5Sget_simple_extent_dims, (dspace, dims, NULL));
398:   *seqlen = (PetscInt)dims[0];
399:   PetscCall(PetscFree(dims));
400:   PetscCallHDF5(H5Dclose, (dset));
401:   PetscCallHDF5(H5Gclose, (group));
402:   PetscFunctionReturn(PETSC_SUCCESS);
403: }

405: static PetscErrorCode DMSequenceView_HDF5(DM dm, const char seqname[], PetscInt seqnum, PetscScalar value, PetscViewer viewer)
406: {
407:   Vec         stamp;
408:   PetscMPIInt rank;

410:   PetscFunctionBegin;
411:   if (seqnum < 0) PetscFunctionReturn(PETSC_SUCCESS);
412:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
413:   PetscCall(VecCreateMPI(PetscObjectComm((PetscObject)viewer), rank ? 0 : 1, 1, &stamp));
414:   PetscCall(VecSetBlockSize(stamp, 1));
415:   PetscCall(PetscObjectSetName((PetscObject)stamp, seqname));
416:   if (rank == 0) {
417:     PetscReal timeScale;
418:     PetscBool istime;

420:     PetscCall(PetscStrncmp(seqname, "time", 5, &istime));
421:     if (istime) {
422:       PetscCall(DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale));
423:       value *= timeScale;
424:     }
425:     PetscCall(VecSetValue(stamp, 0, value, INSERT_VALUES));
426:   }
427:   PetscCall(VecAssemblyBegin(stamp));
428:   PetscCall(VecAssemblyEnd(stamp));
429:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/"));
430:   PetscCall(PetscViewerHDF5PushTimestepping(viewer));
431:   PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum)); /* seqnum < 0 jumps out above */
432:   PetscCall(VecView(stamp, viewer));
433:   PetscCall(PetscViewerHDF5PopTimestepping(viewer));
434:   PetscCall(PetscViewerHDF5PopGroup(viewer));
435:   PetscCall(VecDestroy(&stamp));
436:   PetscFunctionReturn(PETSC_SUCCESS);
437: }

439: PetscErrorCode DMSequenceLoad_HDF5_Internal(DM dm, const char seqname[], PetscInt seqnum, PetscScalar *value, PetscViewer viewer)
440: {
441:   Vec         stamp;
442:   PetscMPIInt rank;

444:   PetscFunctionBegin;
445:   if (seqnum < 0) PetscFunctionReturn(PETSC_SUCCESS);
446:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
447:   PetscCall(VecCreateMPI(PetscObjectComm((PetscObject)viewer), rank ? 0 : 1, 1, &stamp));
448:   PetscCall(VecSetBlockSize(stamp, 1));
449:   PetscCall(PetscObjectSetName((PetscObject)stamp, seqname));
450:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/"));
451:   PetscCall(PetscViewerHDF5PushTimestepping(viewer));
452:   PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum)); /* seqnum < 0 jumps out above */
453:   PetscCall(VecLoad(stamp, viewer));
454:   PetscCall(PetscViewerHDF5PopTimestepping(viewer));
455:   PetscCall(PetscViewerHDF5PopGroup(viewer));
456:   if (rank == 0) {
457:     const PetscScalar *a;
458:     PetscReal          timeScale;
459:     PetscBool          istime;

461:     PetscCall(VecGetArrayRead(stamp, &a));
462:     *value = a[0];
463:     PetscCall(VecRestoreArrayRead(stamp, &a));
464:     PetscCall(PetscStrncmp(seqname, "time", 5, &istime));
465:     if (istime) {
466:       PetscCall(DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale));
467:       *value /= timeScale;
468:     }
469:   }
470:   PetscCall(VecDestroy(&stamp));
471:   PetscFunctionReturn(PETSC_SUCCESS);
472: }

474: static PetscErrorCode DMPlexCreateCutVertexLabel_Private(DM dm, DMLabel cutLabel, DMLabel *cutVertexLabel)
475: {
476:   IS              cutcells = NULL;
477:   const PetscInt *cutc;
478:   PetscInt        cellHeight, vStart, vEnd, cStart, cEnd, c;

480:   PetscFunctionBegin;
481:   if (!cutLabel) PetscFunctionReturn(PETSC_SUCCESS);
482:   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
483:   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
484:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
485:   /* Label vertices that should be duplicated */
486:   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Cut Vertices", cutVertexLabel));
487:   PetscCall(DMLabelGetStratumIS(cutLabel, 2, &cutcells));
488:   if (cutcells) {
489:     PetscInt n;

491:     PetscCall(ISGetIndices(cutcells, &cutc));
492:     PetscCall(ISGetLocalSize(cutcells, &n));
493:     for (c = 0; c < n; ++c) {
494:       if ((cutc[c] >= cStart) && (cutc[c] < cEnd)) {
495:         PetscInt *closure = NULL;
496:         PetscInt  closureSize, cl, value;

498:         PetscCall(DMPlexGetTransitiveClosure(dm, cutc[c], PETSC_TRUE, &closureSize, &closure));
499:         for (cl = 0; cl < closureSize * 2; cl += 2) {
500:           if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) {
501:             PetscCall(DMLabelGetValue(cutLabel, closure[cl], &value));
502:             if (value == 1) PetscCall(DMLabelSetValue(*cutVertexLabel, closure[cl], 1));
503:           }
504:         }
505:         PetscCall(DMPlexRestoreTransitiveClosure(dm, cutc[c], PETSC_TRUE, &closureSize, &closure));
506:       }
507:     }
508:     PetscCall(ISRestoreIndices(cutcells, &cutc));
509:   }
510:   PetscCall(ISDestroy(&cutcells));
511:   PetscFunctionReturn(PETSC_SUCCESS);
512: }

514: PetscErrorCode VecView_Plex_Local_HDF5_Internal(Vec v, PetscViewer viewer)
515: {
516:   DM                dm;
517:   DM                dmBC;
518:   PetscSection      section, sectionGlobal;
519:   Vec               gv;
520:   const char       *name;
521:   PetscViewerFormat format;
522:   PetscInt          seqnum;
523:   PetscReal         seqval;
524:   PetscBool         isseq;

526:   PetscFunctionBegin;
527:   PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
528:   PetscCall(VecGetDM(v, &dm));
529:   PetscCall(DMGetLocalSection(dm, &section));
530:   PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, &seqval));
531:   PetscCall(DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar)seqval, viewer));
532:   if (seqnum >= 0) {
533:     PetscCall(PetscViewerHDF5PushTimestepping(viewer));
534:     PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
535:   }
536:   PetscCall(PetscViewerGetFormat(viewer, &format));
537:   PetscCall(DMGetOutputDM(dm, &dmBC));
538:   PetscCall(DMGetGlobalSection(dmBC, &sectionGlobal));
539:   PetscCall(DMGetGlobalVector(dmBC, &gv));
540:   PetscCall(PetscObjectGetName((PetscObject)v, &name));
541:   PetscCall(PetscObjectSetName((PetscObject)gv, name));
542:   PetscCall(DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv));
543:   PetscCall(DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv));
544:   PetscCall(PetscObjectTypeCompare((PetscObject)gv, VECSEQ, &isseq));
545:   if (format == PETSC_VIEWER_HDF5_VIZ) {
546:     /* Output visualization representation */
547:     PetscInt numFields, f;
548:     DMLabel  cutLabel, cutVertexLabel = NULL;

550:     PetscCall(PetscSectionGetNumFields(section, &numFields));
551:     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
552:     for (f = 0; f < numFields; ++f) {
553:       Vec                     subv;
554:       IS                      is;
555:       const char             *fname, *fgroup, *componentName, *fname_def = "unnamed";
556:       char                    subname[PETSC_MAX_PATH_LEN];
557:       PetscInt                Nc, c;
558:       PetscInt                pStart, pEnd;
559:       PetscViewerVTKFieldType ft;

561:       PetscCall(DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft));
562:       if (ft == PETSC_VTK_INVALID) continue;
563:       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
564:       PetscCall(PetscSectionGetFieldName(section, f, &fname));
565:       if (!fname) fname = fname_def;

567:       PetscCall(PetscViewerHDF5PushGroup(viewer, fgroup));

569:       if (cutLabel) {
570:         const PetscScalar *ga;
571:         PetscScalar       *suba;
572:         PetscInt           gstart, subSize = 0, extSize = 0, subOff = 0, newOff = 0, p;

574:         PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
575:         PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
576:         for (p = pStart; p < pEnd; ++p) {
577:           PetscInt gdof, fdof = 0, val;

579:           PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
580:           if (gdof > 0) PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
581:           subSize += fdof;
582:           PetscCall(DMLabelGetValue(cutVertexLabel, p, &val));
583:           if (val == 1) extSize += fdof;
584:         }
585:         PetscCall(VecCreate(PetscObjectComm((PetscObject)gv), &subv));
586:         PetscCall(VecSetSizes(subv, subSize + extSize, PETSC_DETERMINE));
587:         PetscCall(VecSetBlockSize(subv, Nc));
588:         PetscCall(VecSetType(subv, VECSTANDARD));
589:         PetscCall(VecGetOwnershipRange(gv, &gstart, NULL));
590:         PetscCall(VecGetArrayRead(gv, &ga));
591:         PetscCall(VecGetArray(subv, &suba));
592:         for (p = pStart; p < pEnd; ++p) {
593:           PetscInt gdof, goff, val;

595:           PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
596:           if (gdof > 0) {
597:             PetscInt fdof, fc, f2, poff = 0;

599:             PetscCall(PetscSectionGetOffset(sectionGlobal, p, &goff));
600:             /* Can get rid of this loop by storing field information in the global section */
601:             for (f2 = 0; f2 < f; ++f2) {
602:               PetscCall(PetscSectionGetFieldDof(section, p, f2, &fdof));
603:               poff += fdof;
604:             }
605:             PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
606:             for (fc = 0; fc < fdof; ++fc, ++subOff) suba[subOff] = ga[goff + poff + fc - gstart];
607:             PetscCall(DMLabelGetValue(cutVertexLabel, p, &val));
608:             if (val == 1) {
609:               for (fc = 0; fc < fdof; ++fc, ++newOff) suba[subSize + newOff] = ga[goff + poff + fc - gstart];
610:             }
611:           }
612:         }
613:         PetscCall(VecRestoreArrayRead(gv, &ga));
614:         PetscCall(VecRestoreArray(subv, &suba));
615:         PetscCall(DMLabelDestroy(&cutVertexLabel));
616:       } else {
617:         PetscCall(PetscSectionGetField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv));
618:       }
619:       PetscCall(PetscStrncpy(subname, name, sizeof(subname)));
620:       PetscCall(PetscStrlcat(subname, "_", sizeof(subname)));
621:       PetscCall(PetscStrlcat(subname, fname, sizeof(subname)));
622:       PetscCall(PetscObjectSetName((PetscObject)subv, subname));
623:       if (isseq) PetscCall(VecView_Seq(subv, viewer));
624:       else PetscCall(VecView_MPI(subv, viewer));
625:       if (ft == PETSC_VTK_POINT_VECTOR_FIELD || ft == PETSC_VTK_CELL_VECTOR_FIELD) {
626:         PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, "vector_field_type", PETSC_STRING, "vector"));
627:       } else {
628:         PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, "vector_field_type", PETSC_STRING, "scalar"));
629:       }

631:       /* Output the component names in the field if available */
632:       PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
633:       for (c = 0; c < Nc; ++c) {
634:         char componentNameLabel[PETSC_MAX_PATH_LEN];
635:         PetscCall(PetscSectionGetComponentName(section, f, c, &componentName));
636:         PetscCall(PetscSNPrintf(componentNameLabel, sizeof(componentNameLabel), "componentName%" PetscInt_FMT, c));
637:         PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, componentNameLabel, PETSC_STRING, componentName));
638:       }

640:       if (cutLabel) PetscCall(VecDestroy(&subv));
641:       else PetscCall(PetscSectionRestoreField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv));
642:       PetscCall(PetscViewerHDF5PopGroup(viewer));
643:     }
644:   } else {
645:     /* Output full vector */
646:     PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
647:     if (isseq) PetscCall(VecView_Seq(gv, viewer));
648:     else PetscCall(VecView_MPI(gv, viewer));
649:     PetscCall(PetscViewerHDF5PopGroup(viewer));
650:   }
651:   if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
652:   PetscCall(DMRestoreGlobalVector(dmBC, &gv));
653:   PetscFunctionReturn(PETSC_SUCCESS);
654: }

656: PetscErrorCode VecView_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
657: {
658:   DMPlexStorageVersion version;
659:   DM                   dm;
660:   Vec                  locv;
661:   PetscObject          isZero;
662:   const char          *name;
663:   PetscReal            time;

665:   PetscFunctionBegin;
666:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
667:   PetscCall(PetscInfo(v, "Writing Vec %s storage version %d.%d.%d\n", v->hdr.name, version->major, version->minor, version->subminor));

669:   PetscCall(VecGetDM(v, &dm));
670:   PetscCall(DMGetLocalVector(dm, &locv));
671:   PetscCall(PetscObjectGetName((PetscObject)v, &name));
672:   PetscCall(PetscObjectSetName((PetscObject)locv, name));
673:   PetscCall(PetscObjectQuery((PetscObject)v, "__Vec_bc_zero__", &isZero));
674:   PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", isZero));
675:   PetscCall(DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv));
676:   PetscCall(DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv));
677:   PetscCall(DMGetOutputSequenceNumber(dm, NULL, &time));
678:   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL));
679:   PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
680:   if (DMPlexStorageVersionEQ(version, 1, 1, 0)) {
681:     PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
682:     PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ));
683:     PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
684:     PetscCall(PetscViewerPopFormat(viewer));
685:     PetscCall(PetscViewerHDF5PopGroup(viewer));
686:   }
687:   PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", NULL));
688:   PetscCall(DMRestoreLocalVector(dm, &locv));
689:   PetscFunctionReturn(PETSC_SUCCESS);
690: }

692: PetscErrorCode VecView_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
693: {
694:   PetscBool isseq;

696:   PetscFunctionBegin;
697:   PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
698:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
699:   if (isseq) PetscCall(VecView_Seq(v, viewer));
700:   else PetscCall(VecView_MPI(v, viewer));
701:   PetscCall(PetscViewerHDF5PopGroup(viewer));
702:   PetscFunctionReturn(PETSC_SUCCESS);
703: }

705: PetscErrorCode VecLoad_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
706: {
707:   DM          dm;
708:   Vec         locv;
709:   const char *name;
710:   PetscInt    seqnum;

712:   PetscFunctionBegin;
713:   PetscCall(VecGetDM(v, &dm));
714:   PetscCall(DMGetLocalVector(dm, &locv));
715:   PetscCall(PetscObjectGetName((PetscObject)v, &name));
716:   PetscCall(PetscObjectSetName((PetscObject)locv, name));
717:   PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, NULL));
718:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
719:   if (seqnum >= 0) {
720:     PetscCall(PetscViewerHDF5PushTimestepping(viewer));
721:     PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
722:   }
723:   PetscCall(VecLoad_Plex_Local(locv, viewer));
724:   if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
725:   PetscCall(PetscViewerHDF5PopGroup(viewer));
726:   PetscCall(DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v));
727:   PetscCall(DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v));
728:   PetscCall(DMRestoreLocalVector(dm, &locv));
729:   PetscFunctionReturn(PETSC_SUCCESS);
730: }

732: PetscErrorCode VecLoad_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
733: {
734:   DM       dm;
735:   PetscInt seqnum;

737:   PetscFunctionBegin;
738:   PetscCall(VecGetDM(v, &dm));
739:   PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, NULL));
740:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
741:   if (seqnum >= 0) {
742:     PetscCall(PetscViewerHDF5PushTimestepping(viewer));
743:     PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
744:   }
745:   PetscCall(VecLoad_Default(v, viewer));
746:   if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
747:   PetscCall(PetscViewerHDF5PopGroup(viewer));
748:   PetscFunctionReturn(PETSC_SUCCESS);
749: }

751: static PetscErrorCode DMPlexDistributionView_HDF5_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
752: {
753:   MPI_Comm           comm;
754:   PetscMPIInt        size, rank;
755:   PetscInt           size_petsc_int;
756:   const char        *topologydm_name, *distribution_name;
757:   const PetscInt    *gpoint;
758:   PetscInt           pStart, pEnd, p;
759:   PetscSF            pointSF;
760:   PetscInt           nroots, nleaves;
761:   const PetscInt    *ilocal;
762:   const PetscSFNode *iremote;
763:   IS                 chartSizesIS, ownersIS, gpointsIS;
764:   PetscInt          *chartSize, *owners, *gpoints;

766:   PetscFunctionBegin;
767:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
768:   PetscCallMPI(MPI_Comm_size(comm, &size));
769:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
770:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
771:   PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
772:   if (!distribution_name) PetscFunctionReturn(PETSC_SUCCESS);
773:   PetscCall(PetscLogEventBegin(DMPLEX_DistributionView, viewer, 0, 0, 0));
774:   size_petsc_int = (PetscInt)size;
775:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "comm_size", PETSC_INT, (void *)&size_petsc_int));
776:   PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
777:   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
778:   PetscCall(PetscMalloc1(1, &chartSize));
779:   *chartSize = pEnd - pStart;
780:   PetscCall(PetscMalloc1(*chartSize, &owners));
781:   PetscCall(PetscMalloc1(*chartSize, &gpoints));
782:   PetscCall(DMGetPointSF(dm, &pointSF));
783:   PetscCall(PetscSFGetGraph(pointSF, &nroots, &nleaves, &ilocal, &iremote));
784:   for (p = pStart; p < pEnd; ++p) {
785:     PetscInt gp = gpoint[p - pStart];

787:     owners[p - pStart]  = rank;
788:     gpoints[p - pStart] = (gp < 0 ? -(gp + 1) : gp);
789:   }
790:   for (p = 0; p < nleaves; ++p) {
791:     PetscInt ilocalp = (ilocal ? ilocal[p] : p);

793:     owners[ilocalp] = iremote[p].rank;
794:   }
795:   PetscCall(ISCreateGeneral(comm, 1, chartSize, PETSC_OWN_POINTER, &chartSizesIS));
796:   PetscCall(ISCreateGeneral(comm, *chartSize, owners, PETSC_OWN_POINTER, &ownersIS));
797:   PetscCall(ISCreateGeneral(comm, *chartSize, gpoints, PETSC_OWN_POINTER, &gpointsIS));
798:   PetscCall(PetscObjectSetName((PetscObject)chartSizesIS, "chart_sizes"));
799:   PetscCall(PetscObjectSetName((PetscObject)ownersIS, "owners"));
800:   PetscCall(PetscObjectSetName((PetscObject)gpointsIS, "global_point_numbers"));
801:   PetscCall(ISView(chartSizesIS, viewer));
802:   PetscCall(ISView(ownersIS, viewer));
803:   PetscCall(ISView(gpointsIS, viewer));
804:   PetscCall(ISDestroy(&chartSizesIS));
805:   PetscCall(ISDestroy(&ownersIS));
806:   PetscCall(ISDestroy(&gpointsIS));
807:   PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
808:   PetscCall(PetscLogEventEnd(DMPLEX_DistributionView, viewer, 0, 0, 0));
809:   PetscFunctionReturn(PETSC_SUCCESS);
810: }

812: static PetscErrorCode DMPlexTopologyView_HDF5_Inner_Private(DM dm, IS globalPointNumbers, PetscViewer viewer, PetscInt pStart, PetscInt pEnd, const char pointsName[], const char coneSizesName[], const char conesName[], const char orientationsName[])
813: {
814:   IS              coneSizesIS, conesIS, orientationsIS;
815:   PetscInt       *coneSizes, *cones, *orientations;
816:   const PetscInt *gpoint;
817:   PetscInt        nPoints = 0, conesSize = 0;
818:   PetscInt        p, c, s;
819:   MPI_Comm        comm;

821:   PetscFunctionBegin;
822:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
823:   PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
824:   for (p = pStart; p < pEnd; ++p) {
825:     if (gpoint[p] >= 0) {
826:       PetscInt coneSize;

828:       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
829:       nPoints += 1;
830:       conesSize += coneSize;
831:     }
832:   }
833:   PetscCall(PetscMalloc1(nPoints, &coneSizes));
834:   PetscCall(PetscMalloc1(conesSize, &cones));
835:   PetscCall(PetscMalloc1(conesSize, &orientations));
836:   for (p = pStart, c = 0, s = 0; p < pEnd; ++p) {
837:     if (gpoint[p] >= 0) {
838:       const PetscInt *cone, *ornt;
839:       PetscInt        coneSize, cp;

841:       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
842:       PetscCall(DMPlexGetCone(dm, p, &cone));
843:       PetscCall(DMPlexGetConeOrientation(dm, p, &ornt));
844:       coneSizes[s] = coneSize;
845:       for (cp = 0; cp < coneSize; ++cp, ++c) {
846:         cones[c]        = gpoint[cone[cp]] < 0 ? -(gpoint[cone[cp]] + 1) : gpoint[cone[cp]];
847:         orientations[c] = ornt[cp];
848:       }
849:       ++s;
850:     }
851:   }
852:   PetscCheck(s == nPoints, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of points %" PetscInt_FMT " != %" PetscInt_FMT, s, nPoints);
853:   PetscCheck(c == conesSize, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cone points %" PetscInt_FMT " != %" PetscInt_FMT, c, conesSize);
854:   PetscCall(ISCreateGeneral(comm, nPoints, coneSizes, PETSC_OWN_POINTER, &coneSizesIS));
855:   PetscCall(ISCreateGeneral(comm, conesSize, cones, PETSC_OWN_POINTER, &conesIS));
856:   PetscCall(ISCreateGeneral(comm, conesSize, orientations, PETSC_OWN_POINTER, &orientationsIS));
857:   PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
858:   PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
859:   PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
860:   PetscCall(ISView(coneSizesIS, viewer));
861:   PetscCall(ISView(conesIS, viewer));
862:   PetscCall(ISView(orientationsIS, viewer));
863:   PetscCall(ISDestroy(&coneSizesIS));
864:   PetscCall(ISDestroy(&conesIS));
865:   PetscCall(ISDestroy(&orientationsIS));
866:   if (pointsName) {
867:     IS        pointsIS;
868:     PetscInt *points;

870:     PetscCall(PetscMalloc1(nPoints, &points));
871:     for (p = pStart, c = 0, s = 0; p < pEnd; ++p) {
872:       if (gpoint[p] >= 0) {
873:         points[s] = gpoint[p];
874:         ++s;
875:       }
876:     }
877:     PetscCall(ISCreateGeneral(comm, nPoints, points, PETSC_OWN_POINTER, &pointsIS));
878:     PetscCall(PetscObjectSetName((PetscObject)pointsIS, pointsName));
879:     PetscCall(ISView(pointsIS, viewer));
880:     PetscCall(ISDestroy(&pointsIS));
881:   }
882:   PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
883:   PetscFunctionReturn(PETSC_SUCCESS);
884: }

886: static PetscErrorCode DMPlexTopologyView_HDF5_Legacy_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
887: {
888:   const char *pointsName, *coneSizesName, *conesName, *orientationsName;
889:   PetscInt    pStart, pEnd, dim;

891:   PetscFunctionBegin;
892:   pointsName       = "order";
893:   coneSizesName    = "cones";
894:   conesName        = "cells";
895:   orientationsName = "orientation";
896:   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
897:   PetscCall(DMPlexTopologyView_HDF5_Inner_Private(dm, globalPointNumbers, viewer, pStart, pEnd, pointsName, coneSizesName, conesName, orientationsName));
898:   PetscCall(DMGetDimension(dm, &dim));
899:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, conesName, "cell_dim", PETSC_INT, (void *)&dim));
900:   PetscFunctionReturn(PETSC_SUCCESS);
901: }

903: //TODO get this numbering right away without needing this function
904: /* Renumber global point numbers so that they are 0-based per stratum */
905: static PetscErrorCode RenumberGlobalPointNumbersPerStratum_Private(DM dm, IS globalPointNumbers, IS *newGlobalPointNumbers, IS *strataPermutation)
906: {
907:   PetscInt        d, depth, p, n;
908:   PetscInt       *offsets;
909:   const PetscInt *gpn;
910:   PetscInt       *ngpn;
911:   MPI_Comm        comm;

913:   PetscFunctionBegin;
914:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
915:   PetscCall(ISGetLocalSize(globalPointNumbers, &n));
916:   PetscCall(ISGetIndices(globalPointNumbers, &gpn));
917:   PetscCall(PetscMalloc1(n, &ngpn));
918:   PetscCall(DMPlexGetDepth(dm, &depth));
919:   PetscCall(PetscMalloc1(depth + 1, &offsets));
920:   for (d = 0; d <= depth; d++) {
921:     PetscInt pStart, pEnd;

923:     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
924:     offsets[d] = PETSC_INT_MAX;
925:     for (p = pStart; p < pEnd; p++) {
926:       if (gpn[p] >= 0 && gpn[p] < offsets[d]) offsets[d] = gpn[p];
927:     }
928:   }
929:   PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, offsets, depth + 1, MPIU_INT, MPI_MIN, comm));
930:   for (d = 0; d <= depth; d++) {
931:     PetscInt pStart, pEnd;

933:     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
934:     for (p = pStart; p < pEnd; p++) ngpn[p] = gpn[p] - PetscSign(gpn[p]) * offsets[d];
935:   }
936:   PetscCall(ISRestoreIndices(globalPointNumbers, &gpn));
937:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)globalPointNumbers), n, ngpn, PETSC_OWN_POINTER, newGlobalPointNumbers));
938:   {
939:     PetscInt *perm;

941:     PetscCall(PetscMalloc1(depth + 1, &perm));
942:     for (d = 0; d <= depth; d++) perm[d] = d;
943:     PetscCall(PetscSortIntWithPermutation(depth + 1, offsets, perm));
944:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, depth + 1, perm, PETSC_OWN_POINTER, strataPermutation));
945:   }
946:   PetscCall(PetscFree(offsets));
947:   PetscFunctionReturn(PETSC_SUCCESS);
948: }

950: static PetscErrorCode DMPlexTopologyView_HDF5_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
951: {
952:   IS          globalPointNumbers0, strataPermutation;
953:   const char *coneSizesName, *conesName, *orientationsName;
954:   PetscInt    depth, d;
955:   MPI_Comm    comm;

957:   PetscFunctionBegin;
958:   coneSizesName    = "cone_sizes";
959:   conesName        = "cones";
960:   orientationsName = "orientations";
961:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
962:   PetscCall(DMPlexGetDepth(dm, &depth));
963:   {
964:     PetscInt dim;

966:     PetscCall(DMGetDimension(dm, &dim));
967:     PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "cell_dim", PETSC_INT, &dim));
968:     PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "depth", PETSC_INT, &depth));
969:   }

971:   PetscCall(RenumberGlobalPointNumbersPerStratum_Private(dm, globalPointNumbers, &globalPointNumbers0, &strataPermutation));
972:   /* TODO dirty trick to save serial IS using the same parallel viewer */
973:   {
974:     IS              spOnComm;
975:     PetscInt        n   = 0, N;
976:     const PetscInt *idx = NULL;
977:     const PetscInt *old;
978:     PetscMPIInt     rank;

980:     PetscCallMPI(MPI_Comm_rank(comm, &rank));
981:     PetscCall(ISGetLocalSize(strataPermutation, &N));
982:     PetscCall(ISGetIndices(strataPermutation, &old));
983:     if (!rank) {
984:       n   = N;
985:       idx = old;
986:     }
987:     PetscCall(ISCreateGeneral(comm, n, idx, PETSC_COPY_VALUES, &spOnComm));
988:     PetscCall(ISRestoreIndices(strataPermutation, &old));
989:     PetscCall(ISDestroy(&strataPermutation));
990:     strataPermutation = spOnComm;
991:   }
992:   PetscCall(PetscObjectSetName((PetscObject)strataPermutation, "permutation"));
993:   PetscCall(ISView(strataPermutation, viewer));
994:   PetscCall(PetscViewerHDF5PushGroup(viewer, "strata"));
995:   for (d = 0; d <= depth; d++) {
996:     PetscInt pStart, pEnd;
997:     char     group[128];

999:     PetscCall(PetscSNPrintf(group, sizeof(group), "%" PetscInt_FMT, d));
1000:     PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1001:     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
1002:     PetscCall(DMPlexTopologyView_HDF5_Inner_Private(dm, globalPointNumbers0, viewer, pStart, pEnd, NULL, coneSizesName, conesName, orientationsName));
1003:     PetscCall(PetscViewerHDF5PopGroup(viewer));
1004:   }
1005:   PetscCall(PetscViewerHDF5PopGroup(viewer)); /* strata */
1006:   PetscCall(ISDestroy(&globalPointNumbers0));
1007:   PetscCall(ISDestroy(&strataPermutation));
1008:   PetscFunctionReturn(PETSC_SUCCESS);
1009: }

1011: PetscErrorCode DMPlexTopologyView_HDF5_Internal(DM dm, IS globalPointNumbers, PetscViewer viewer)
1012: {
1013:   DMPlexStorageVersion version;
1014:   const char          *topologydm_name;
1015:   char                 group[PETSC_MAX_PATH_LEN];

1017:   PetscFunctionBegin;
1018:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1019:   PetscCall(PetscInfo(dm, "Writing DM %s storage version %d.%d.%d\n", dm->hdr.name, version->major, version->minor, version->subminor));
1020:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1021:   if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1022:     PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s", topologydm_name));
1023:   } else {
1024:     PetscCall(PetscStrncpy(group, "/", sizeof(group)));
1025:   }
1026:   PetscCall(PetscViewerHDF5PushGroup(viewer, group));

1028:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topology"));
1029:   if (version->major < 3) {
1030:     PetscCall(DMPlexTopologyView_HDF5_Legacy_Private(dm, globalPointNumbers, viewer));
1031:   } else {
1032:     /* since DMPlexStorageVersion 3.0.0 */
1033:     PetscCall(DMPlexTopologyView_HDF5_Private(dm, globalPointNumbers, viewer));
1034:   }
1035:   PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "topology" */

1037:   if (DMPlexStorageVersionGE(version, 2, 1, 0)) {
1038:     const char *distribution_name;

1040:     PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
1041:     PetscCall(PetscViewerHDF5PushGroup(viewer, "distributions"));
1042:     PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));
1043:     PetscCall(PetscViewerHDF5PushGroup(viewer, distribution_name));
1044:     PetscCall(DMPlexDistributionView_HDF5_Private(dm, globalPointNumbers, viewer));
1045:     PetscCall(PetscViewerHDF5PopGroup(viewer)); /* distribution_name */
1046:     PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "distributions" */
1047:   }

1049:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1050:   PetscFunctionReturn(PETSC_SUCCESS);
1051: }

1053: static PetscErrorCode CreateConesIS_Private(DM dm, PetscInt cStart, PetscInt cEnd, IS globalCellNumbers, PetscInt *numCorners, IS *cellIS)
1054: {
1055:   PetscSF         sfPoint;
1056:   DMLabel         cutLabel, cutVertexLabel         = NULL;
1057:   IS              globalVertexNumbers, cutvertices = NULL;
1058:   const PetscInt *gcell, *gvertex, *cutverts = NULL;
1059:   PetscInt       *vertices;
1060:   PetscInt        conesSize = 0;
1061:   PetscInt        dim, numCornersLocal = 0, cell, vStart, vEnd, vExtra = 0, v;

1063:   PetscFunctionBegin;
1064:   *numCorners = 0;
1065:   PetscCall(DMGetDimension(dm, &dim));
1066:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1067:   PetscCall(ISGetIndices(globalCellNumbers, &gcell));

1069:   for (cell = cStart; cell < cEnd; ++cell) {
1070:     PetscInt *closure = NULL;
1071:     PetscInt  closureSize, v, Nc = 0;

1073:     if (gcell[cell] < 0) continue;
1074:     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1075:     for (v = 0; v < closureSize * 2; v += 2) {
1076:       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
1077:     }
1078:     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1079:     conesSize += Nc;
1080:     if (!numCornersLocal) numCornersLocal = Nc;
1081:     else if (numCornersLocal != Nc) numCornersLocal = 1;
1082:   }
1083:   PetscCallMPI(MPIU_Allreduce(&numCornersLocal, numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
1084:   PetscCheck(!numCornersLocal || !(numCornersLocal != *numCorners || *numCorners == 0), PETSC_COMM_SELF, PETSC_ERR_SUP, "Visualization topology currently only supports identical cell shapes");
1085:   /* Handle periodic cuts by identifying vertices which should be duplicated */
1086:   PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1087:   PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
1088:   if (cutVertexLabel) PetscCall(DMLabelGetStratumIS(cutVertexLabel, 1, &cutvertices));
1089:   if (cutvertices) {
1090:     PetscCall(ISGetIndices(cutvertices, &cutverts));
1091:     PetscCall(ISGetLocalSize(cutvertices, &vExtra));
1092:   }
1093:   PetscCall(DMGetPointSF(dm, &sfPoint));
1094:   if (cutLabel) {
1095:     const PetscInt    *ilocal;
1096:     const PetscSFNode *iremote;
1097:     PetscInt           nroots, nleaves;

1099:     PetscCall(PetscSFGetGraph(sfPoint, &nroots, &nleaves, &ilocal, &iremote));
1100:     if (nleaves < 0) {
1101:       PetscCall(PetscObjectReference((PetscObject)sfPoint));
1102:     } else {
1103:       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)sfPoint), &sfPoint));
1104:       PetscCall(PetscSFSetGraph(sfPoint, nroots + vExtra, nleaves, (PetscInt *)ilocal, PETSC_COPY_VALUES, (PetscSFNode *)iremote, PETSC_COPY_VALUES));
1105:     }
1106:   } else {
1107:     PetscCall(PetscObjectReference((PetscObject)sfPoint));
1108:   }
1109:   /* Number all vertices */
1110:   PetscCall(DMPlexCreateNumbering_Plex(dm, vStart, vEnd + vExtra, 0, NULL, sfPoint, &globalVertexNumbers));
1111:   PetscCall(PetscSFDestroy(&sfPoint));
1112:   /* Create cones */
1113:   PetscCall(ISGetIndices(globalVertexNumbers, &gvertex));
1114:   PetscCall(PetscMalloc1(conesSize, &vertices));
1115:   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
1116:     PetscInt *closure = NULL;
1117:     PetscInt  closureSize, Nc = 0, p, value = -1;
1118:     PetscBool replace;

1120:     if (gcell[cell] < 0) continue;
1121:     if (cutLabel) PetscCall(DMLabelGetValue(cutLabel, cell, &value));
1122:     replace = (value == 2) ? PETSC_TRUE : PETSC_FALSE;
1123:     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1124:     for (p = 0; p < closureSize * 2; p += 2) {
1125:       if ((closure[p] >= vStart) && (closure[p] < vEnd)) closure[Nc++] = closure[p];
1126:     }
1127:     PetscCall(DMPlexReorderCell(dm, cell, closure));
1128:     for (p = 0; p < Nc; ++p) {
1129:       PetscInt nv, gv = gvertex[closure[p] - vStart];

1131:       if (replace) {
1132:         PetscCall(PetscFindInt(closure[p], vExtra, cutverts, &nv));
1133:         if (nv >= 0) gv = gvertex[vEnd - vStart + nv];
1134:       }
1135:       vertices[v++] = gv < 0 ? -(gv + 1) : gv;
1136:     }
1137:     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1138:   }
1139:   PetscCall(ISRestoreIndices(globalVertexNumbers, &gvertex));
1140:   PetscCall(ISDestroy(&globalVertexNumbers));
1141:   PetscCall(ISRestoreIndices(globalCellNumbers, &gcell));
1142:   if (cutvertices) PetscCall(ISRestoreIndices(cutvertices, &cutverts));
1143:   PetscCall(ISDestroy(&cutvertices));
1144:   PetscCall(DMLabelDestroy(&cutVertexLabel));
1145:   PetscCheck(v == conesSize, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %" PetscInt_FMT " != %" PetscInt_FMT, v, conesSize);
1146:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), conesSize, vertices, PETSC_OWN_POINTER, cellIS));
1147:   PetscCall(PetscLayoutSetBlockSize((*cellIS)->map, *numCorners));
1148:   PetscCall(PetscObjectSetName((PetscObject)*cellIS, "cells"));
1149:   PetscFunctionReturn(PETSC_SUCCESS);
1150: }

1152: static PetscErrorCode DMPlexTopologyView_HDF5_XDMF_Private(DM dm, IS globalCellNumbers, PetscViewer viewer)
1153: {
1154:   DM       cdm;
1155:   DMLabel  depthLabel, ctLabel;
1156:   IS       cellIS;
1157:   PetscInt dim, depth, cellHeight, c, n = 0;

1159:   PetscFunctionBegin;
1160:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz"));
1161:   PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));

1163:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1164:   PetscCall(DMGetDimension(dm, &dim));
1165:   PetscCall(DMPlexGetDepth(dm, &depth));
1166:   PetscCall(DMGetCoordinateDM(dm, &cdm));
1167:   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
1168:   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
1169:   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
1170:   for (c = 0; c < DM_NUM_POLYTOPES; ++c) {
1171:     const DMPolytopeType ict = (DMPolytopeType)c;
1172:     PetscInt             pStart, pEnd, dep, numCorners;
1173:     PetscBool            output = PETSC_FALSE, doOutput;

1175:     if (ict == DM_POLYTOPE_FV_GHOST) continue;
1176:     PetscCall(DMLabelGetStratumBounds(ctLabel, ict, &pStart, &pEnd));
1177:     if (pStart >= 0) {
1178:       PetscCall(DMLabelGetValue(depthLabel, pStart, &dep));
1179:       if (dep == depth - cellHeight) output = PETSC_TRUE;
1180:     }
1181:     PetscCallMPI(MPIU_Allreduce(&output, &doOutput, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
1182:     if (!doOutput) continue;
1183:     PetscCall(CreateConesIS_Private(dm, pStart, pEnd, globalCellNumbers, &numCorners, &cellIS));
1184:     if (!n) {
1185:       PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz/topology"));
1186:     } else {
1187:       char group[PETSC_MAX_PATH_LEN];

1189:       PetscCall(PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/viz/topology_%" PetscInt_FMT, n));
1190:       PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1191:     }
1192:     PetscCall(ISView(cellIS, viewer));
1193:     PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)cellIS, "cell_corners", PETSC_INT, (void *)&numCorners));
1194:     PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)cellIS, "cell_dim", PETSC_INT, (void *)&dim));
1195:     PetscCall(ISDestroy(&cellIS));
1196:     PetscCall(PetscViewerHDF5PopGroup(viewer));
1197:     ++n;
1198:   }
1199:   PetscFunctionReturn(PETSC_SUCCESS);
1200: }

1202: static PetscErrorCode DMPlexCoordinatesView_HDF5_Legacy_Private(DM dm, PetscViewer viewer)
1203: {
1204:   DM        cdm;
1205:   Vec       coordinates, newcoords;
1206:   PetscReal lengthScale;
1207:   PetscInt  m, M, bs;

1209:   PetscFunctionBegin;
1210:   PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1211:   PetscCall(DMGetCoordinateDM(dm, &cdm));
1212:   PetscCall(DMGetCoordinates(dm, &coordinates));
1213:   PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &newcoords));
1214:   PetscCall(PetscObjectSetName((PetscObject)newcoords, "vertices"));
1215:   PetscCall(VecGetSize(coordinates, &M));
1216:   PetscCall(VecGetLocalSize(coordinates, &m));
1217:   PetscCall(VecSetSizes(newcoords, m, M));
1218:   PetscCall(VecGetBlockSize(coordinates, &bs));
1219:   PetscCall(VecSetBlockSize(newcoords, bs));
1220:   PetscCall(VecSetType(newcoords, VECSTANDARD));
1221:   PetscCall(VecCopy(coordinates, newcoords));
1222:   PetscCall(VecScale(newcoords, lengthScale));
1223:   /* Did not use DMGetGlobalVector() in order to bypass default group assignment */
1224:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/geometry"));
1225:   PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
1226:   PetscCall(VecView(newcoords, viewer));
1227:   PetscCall(PetscViewerPopFormat(viewer));
1228:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1229:   PetscCall(VecDestroy(&newcoords));
1230:   PetscFunctionReturn(PETSC_SUCCESS);
1231: }

1233: PetscErrorCode DMPlexCoordinatesView_HDF5_Internal(DM dm, PetscViewer viewer)
1234: {
1235:   DM          cdm;
1236:   Vec         coords, newcoords;
1237:   PetscInt    m, M, bs;
1238:   PetscReal   lengthScale;
1239:   PetscBool   viewSection = PETSC_TRUE;
1240:   const char *topologydm_name, *coordinatedm_name, *coordinates_name;

1242:   PetscFunctionBegin;
1243:   {
1244:     PetscViewerFormat    format;
1245:     DMPlexStorageVersion version;

1247:     PetscCall(PetscViewerGetFormat(viewer, &format));
1248:     PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1249:     if (!DMPlexStorageVersionGE(version, 2, 0, 0) || format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1250:       PetscCall(DMPlexCoordinatesView_HDF5_Legacy_Private(dm, viewer));
1251:       PetscFunctionReturn(PETSC_SUCCESS);
1252:     }
1253:   }
1254:   /* since 2.0.0 */
1255:   PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_view_coordinate_section", &viewSection, NULL));
1256:   PetscCall(DMGetCoordinateDM(dm, &cdm));
1257:   PetscCall(DMGetCoordinates(dm, &coords));
1258:   PetscCall(PetscObjectGetName((PetscObject)cdm, &coordinatedm_name));
1259:   PetscCall(PetscObjectGetName((PetscObject)coords, &coordinates_name));
1260:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1261:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1262:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1263:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "coordinateDMName", PETSC_STRING, coordinatedm_name));
1264:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "coordinatesName", PETSC_STRING, coordinates_name));
1265:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1266:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1267:   if (viewSection) PetscCall(DMPlexSectionView(dm, viewer, cdm));
1268:   PetscCall(VecCreate(PetscObjectComm((PetscObject)coords), &newcoords));
1269:   PetscCall(PetscObjectSetName((PetscObject)newcoords, coordinates_name));
1270:   PetscCall(VecGetSize(coords, &M));
1271:   PetscCall(VecGetLocalSize(coords, &m));
1272:   PetscCall(VecSetSizes(newcoords, m, M));
1273:   PetscCall(VecGetBlockSize(coords, &bs));
1274:   PetscCall(VecSetBlockSize(newcoords, bs));
1275:   PetscCall(VecSetType(newcoords, VECSTANDARD));
1276:   PetscCall(VecCopy(coords, newcoords));
1277:   PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1278:   PetscCall(VecScale(newcoords, lengthScale));
1279:   PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
1280:   PetscCall(DMPlexGlobalVectorView(dm, viewer, cdm, newcoords));
1281:   PetscCall(PetscViewerPopFormat(viewer));
1282:   PetscCall(VecDestroy(&newcoords));
1283:   PetscFunctionReturn(PETSC_SUCCESS);
1284: }

1286: static PetscErrorCode DMPlexCoordinatesView_HDF5_XDMF_Private(DM dm, PetscViewer viewer)
1287: {
1288:   DM               cdm;
1289:   Vec              coordinatesLocal, newcoords;
1290:   PetscSection     cSection, cGlobalSection;
1291:   PetscScalar     *coords, *ncoords;
1292:   DMLabel          cutLabel, cutVertexLabel = NULL;
1293:   const PetscReal *L;
1294:   PetscReal        lengthScale;
1295:   PetscInt         vStart, vEnd, v, bs, N, coordSize, dof, off, d;
1296:   PetscBool        localized, embedded;

1298:   PetscFunctionBegin;
1299:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1300:   PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1301:   PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
1302:   PetscCall(VecGetBlockSize(coordinatesLocal, &bs));
1303:   PetscCall(DMGetCoordinatesLocalized(dm, &localized));
1304:   if (localized == PETSC_FALSE) PetscFunctionReturn(PETSC_SUCCESS);
1305:   PetscCall(DMGetPeriodicity(dm, NULL, NULL, &L));
1306:   PetscCall(DMGetCoordinateDM(dm, &cdm));
1307:   PetscCall(DMGetLocalSection(cdm, &cSection));
1308:   PetscCall(DMGetGlobalSection(cdm, &cGlobalSection));
1309:   PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1310:   N = 0;

1312:   PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
1313:   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &newcoords));
1314:   PetscCall(PetscSectionGetDof(cSection, vStart, &dof));
1315:   PetscCall(PetscPrintf(PETSC_COMM_SELF, "DOF: %" PetscInt_FMT "\n", dof));
1316:   embedded = (PetscBool)(L && dof == 2 && !cutLabel);
1317:   if (cutVertexLabel) {
1318:     PetscCall(DMLabelGetStratumSize(cutVertexLabel, 1, &v));
1319:     N += dof * v;
1320:   }
1321:   for (v = vStart; v < vEnd; ++v) {
1322:     PetscCall(PetscSectionGetDof(cGlobalSection, v, &dof));
1323:     if (dof < 0) continue;
1324:     if (embedded) N += dof + 1;
1325:     else N += dof;
1326:   }
1327:   if (embedded) PetscCall(VecSetBlockSize(newcoords, bs + 1));
1328:   else PetscCall(VecSetBlockSize(newcoords, bs));
1329:   PetscCall(VecSetSizes(newcoords, N, PETSC_DETERMINE));
1330:   PetscCall(VecSetType(newcoords, VECSTANDARD));
1331:   PetscCall(VecGetArray(coordinatesLocal, &coords));
1332:   PetscCall(VecGetArray(newcoords, &ncoords));
1333:   coordSize = 0;
1334:   for (v = vStart; v < vEnd; ++v) {
1335:     PetscCall(PetscSectionGetDof(cGlobalSection, v, &dof));
1336:     PetscCall(PetscSectionGetOffset(cSection, v, &off));
1337:     if (dof < 0) continue;
1338:     if (embedded) {
1339:       if (L && (L[0] > 0.0) && (L[1] > 0.0)) {
1340:         PetscReal theta, phi, r, R;
1341:         /* XY-periodic */
1342:         /* Suppose its an y-z circle, then
1343:              \hat r = (0, cos(th), sin(th)) \hat x = (1, 0, 0)
1344:            and the circle in that plane is
1345:              \hat r cos(phi) + \hat x sin(phi) */
1346:         theta                = 2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1];
1347:         phi                  = 2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0];
1348:         r                    = L[0] / (2.0 * PETSC_PI * 2.0 * L[1]);
1349:         R                    = L[1] / (2.0 * PETSC_PI);
1350:         ncoords[coordSize++] = PetscSinReal(phi) * r;
1351:         ncoords[coordSize++] = -PetscCosReal(theta) * (R + r * PetscCosReal(phi));
1352:         ncoords[coordSize++] = PetscSinReal(theta) * (R + r * PetscCosReal(phi));
1353:       } else if (L && (L[0] > 0.0)) {
1354:         /* X-periodic */
1355:         ncoords[coordSize++] = -PetscCosReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0]) * (L[0] / (2.0 * PETSC_PI));
1356:         ncoords[coordSize++] = coords[off + 1];
1357:         ncoords[coordSize++] = PetscSinReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0]) * (L[0] / (2.0 * PETSC_PI));
1358:       } else if (L && (L[1] > 0.0)) {
1359:         /* Y-periodic */
1360:         ncoords[coordSize++] = coords[off + 0];
1361:         ncoords[coordSize++] = PetscSinReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1]) * (L[1] / (2.0 * PETSC_PI));
1362:         ncoords[coordSize++] = -PetscCosReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1]) * (L[1] / (2.0 * PETSC_PI));
1363:   #if 0
1364:       } else if ((bd[0] == DM_BOUNDARY_TWIST)) {
1365:         PetscReal phi, r, R;
1366:         /* Mobius strip */
1367:         /* Suppose its an x-z circle, then
1368:              \hat r = (-cos(phi), 0, sin(phi)) \hat y = (0, 1, 0)
1369:            and in that plane we rotate by pi as we go around the circle
1370:              \hat r cos(phi/2) + \hat y sin(phi/2) */
1371:         phi   = 2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0];
1372:         R     = L[0];
1373:         r     = PetscRealPart(coords[off+1]) - L[1]/2.0;
1374:         ncoords[coordSize++] = -PetscCosReal(phi) * (R + r * PetscCosReal(phi/2.0));
1375:         ncoords[coordSize++] =  PetscSinReal(phi/2.0) * r;
1376:         ncoords[coordSize++] =  PetscSinReal(phi) * (R + r * PetscCosReal(phi/2.0));
1377:   #endif
1378:       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Cannot handle periodicity in this domain");
1379:     } else {
1380:       for (d = 0; d < dof; ++d, ++coordSize) ncoords[coordSize] = coords[off + d];
1381:     }
1382:   }
1383:   if (cutVertexLabel) {
1384:     IS              vertices;
1385:     const PetscInt *verts;
1386:     PetscInt        n;

1388:     PetscCall(DMLabelGetStratumIS(cutVertexLabel, 1, &vertices));
1389:     if (vertices) {
1390:       PetscCall(ISGetIndices(vertices, &verts));
1391:       PetscCall(ISGetLocalSize(vertices, &n));
1392:       for (v = 0; v < n; ++v) {
1393:         PetscCall(PetscSectionGetDof(cSection, verts[v], &dof));
1394:         PetscCall(PetscSectionGetOffset(cSection, verts[v], &off));
1395:         for (d = 0; d < dof; ++d) ncoords[coordSize++] = coords[off + d] + ((L[d] > 0.) ? L[d] : 0.0);
1396:       }
1397:       PetscCall(ISRestoreIndices(vertices, &verts));
1398:       PetscCall(ISDestroy(&vertices));
1399:     }
1400:   }
1401:   PetscCheck(coordSize == N, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched sizes: %" PetscInt_FMT " != %" PetscInt_FMT, coordSize, N);
1402:   PetscCall(DMLabelDestroy(&cutVertexLabel));
1403:   PetscCall(VecRestoreArray(coordinatesLocal, &coords));
1404:   PetscCall(VecRestoreArray(newcoords, &ncoords));
1405:   PetscCall(PetscObjectSetName((PetscObject)newcoords, "vertices"));
1406:   PetscCall(VecScale(newcoords, lengthScale));
1407:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz"));
1408:   PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));
1409:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1410:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz/geometry"));
1411:   PetscCall(VecView(newcoords, viewer));
1412:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1413:   PetscCall(VecDestroy(&newcoords));
1414:   PetscFunctionReturn(PETSC_SUCCESS);
1415: }

1417: PetscErrorCode DMPlexLabelsView_HDF5_Internal(DM dm, IS globalPointNumbers, PetscViewer viewer)
1418: {
1419:   const char          *topologydm_name;
1420:   const PetscInt      *gpoint;
1421:   PetscInt             numLabels;
1422:   PetscBool            omitCelltypes = PETSC_FALSE;
1423:   DMPlexStorageVersion version;
1424:   char                 group[PETSC_MAX_PATH_LEN];

1426:   PetscFunctionBegin;
1427:   PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_omit_celltypes", &omitCelltypes, NULL));
1428:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1429:   PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
1430:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1431:   if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1432:     PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s/labels", topologydm_name));
1433:   } else {
1434:     PetscCall(PetscStrncpy(group, "/labels", sizeof(group)));
1435:   }
1436:   PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1437:   PetscCall(DMGetNumLabels(dm, &numLabels));
1438:   for (PetscInt l = 0; l < numLabels; ++l) {
1439:     DMLabel         label;
1440:     const char     *name;
1441:     IS              valueIS, pvalueIS, globalValueIS;
1442:     const PetscInt *values;
1443:     PetscInt        numValues, v;
1444:     PetscBool       isDepth, isCelltype, output;

1446:     PetscCall(DMGetLabelByNum(dm, l, &label));
1447:     PetscCall(PetscObjectGetName((PetscObject)label, &name));
1448:     PetscCall(DMGetLabelOutput(dm, name, &output));
1449:     PetscCall(PetscStrncmp(name, "depth", 10, &isDepth));
1450:     PetscCall(PetscStrncmp(name, "celltype", 10, &isCelltype));
1451:     // TODO Should only filter out celltype if it can be calculated
1452:     if (isDepth || (isCelltype && omitCelltypes) || !output) continue;
1453:     PetscCall(PetscViewerHDF5PushGroup(viewer, name));
1454:     PetscCall(DMLabelGetValueIS(label, &valueIS));
1455:     /* Must copy to a new IS on the global comm */
1456:     PetscCall(ISGetLocalSize(valueIS, &numValues));
1457:     PetscCall(ISGetIndices(valueIS, &values));
1458:     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), numValues, values, PETSC_COPY_VALUES, &pvalueIS));
1459:     PetscCall(ISRestoreIndices(valueIS, &values));
1460:     PetscCall(ISAllGather(pvalueIS, &globalValueIS));
1461:     PetscCall(ISDestroy(&pvalueIS));
1462:     PetscCall(ISSortRemoveDups(globalValueIS));
1463:     PetscCall(ISGetLocalSize(globalValueIS, &numValues));
1464:     PetscCall(ISGetIndices(globalValueIS, &values));
1465:     for (v = 0; v < numValues; ++v) {
1466:       IS              stratumIS, globalStratumIS;
1467:       const PetscInt *spoints = NULL;
1468:       PetscInt       *gspoints, n = 0, gn, p;
1469:       const char     *iname = "indices";
1470:       char            group[PETSC_MAX_PATH_LEN];

1472:       PetscCall(PetscSNPrintf(group, sizeof(group), "%" PetscInt_FMT, values[v]));
1473:       PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1474:       PetscCall(DMLabelGetStratumIS(label, values[v], &stratumIS));

1476:       if (stratumIS) PetscCall(ISGetLocalSize(stratumIS, &n));
1477:       if (stratumIS) PetscCall(ISGetIndices(stratumIS, &spoints));
1478:       for (gn = 0, p = 0; p < n; ++p)
1479:         if (gpoint[spoints[p]] >= 0) ++gn;
1480:       PetscCall(PetscMalloc1(gn, &gspoints));
1481:       for (gn = 0, p = 0; p < n; ++p)
1482:         if (gpoint[spoints[p]] >= 0) gspoints[gn++] = gpoint[spoints[p]];
1483:       if (stratumIS) PetscCall(ISRestoreIndices(stratumIS, &spoints));
1484:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), gn, gspoints, PETSC_OWN_POINTER, &globalStratumIS));
1485:       PetscCall(PetscObjectSetName((PetscObject)globalStratumIS, iname));

1487:       PetscCall(ISView(globalStratumIS, viewer));
1488:       PetscCall(ISDestroy(&globalStratumIS));
1489:       PetscCall(ISDestroy(&stratumIS));
1490:       PetscCall(PetscViewerHDF5PopGroup(viewer));
1491:     }
1492:     PetscCall(ISRestoreIndices(globalValueIS, &values));
1493:     PetscCall(ISDestroy(&globalValueIS));
1494:     PetscCall(ISDestroy(&valueIS));
1495:     PetscCall(PetscViewerHDF5PopGroup(viewer));
1496:   }
1497:   PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
1498:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1499:   PetscFunctionReturn(PETSC_SUCCESS);
1500: }

1502: /* We only write cells and vertices. Does this screw up parallel reading? */
1503: PetscErrorCode DMPlexView_HDF5_Internal(DM dm, PetscViewer viewer)
1504: {
1505:   IS                globalPointNumbers;
1506:   PetscViewerFormat format;
1507:   PetscBool         viz_geom = PETSC_FALSE, xdmf_topo = PETSC_FALSE, petsc_topo = PETSC_FALSE;

1509:   PetscFunctionBegin;
1510:   PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbers));
1511:   PetscCall(DMPlexCoordinatesView_HDF5_Internal(dm, viewer));

1513:   PetscCall(PetscViewerGetFormat(viewer, &format));
1514:   switch (format) {
1515:   case PETSC_VIEWER_HDF5_VIZ:
1516:     viz_geom  = PETSC_TRUE;
1517:     xdmf_topo = PETSC_TRUE;
1518:     break;
1519:   case PETSC_VIEWER_HDF5_XDMF:
1520:     xdmf_topo = PETSC_TRUE;
1521:     break;
1522:   case PETSC_VIEWER_HDF5_PETSC:
1523:     petsc_topo = PETSC_TRUE;
1524:     break;
1525:   case PETSC_VIEWER_DEFAULT:
1526:   case PETSC_VIEWER_NATIVE:
1527:     viz_geom   = PETSC_TRUE;
1528:     xdmf_topo  = PETSC_TRUE;
1529:     petsc_topo = PETSC_TRUE;
1530:     break;
1531:   default:
1532:     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1533:   }

1535:   if (viz_geom) PetscCall(DMPlexCoordinatesView_HDF5_XDMF_Private(dm, viewer));
1536:   if (xdmf_topo) PetscCall(DMPlexTopologyView_HDF5_XDMF_Private(dm, globalPointNumbers, viewer));
1537:   if (petsc_topo) {
1538:     PetscBool viewLabels = PETSC_TRUE;

1540:     PetscCall(DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbers, viewer));
1541:     PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_view_labels", &viewLabels, NULL));
1542:     if (viewLabels) PetscCall(DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbers, viewer));
1543:   }

1545:   PetscCall(ISDestroy(&globalPointNumbers));
1546:   PetscFunctionReturn(PETSC_SUCCESS);
1547: }

1549: PetscErrorCode DMPlexSectionView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm)
1550: {
1551:   MPI_Comm     comm;
1552:   const char  *topologydm_name;
1553:   const char  *sectiondm_name;
1554:   PetscSection gsection;

1556:   PetscFunctionBegin;
1557:   PetscCall(PetscObjectGetComm((PetscObject)sectiondm, &comm));
1558:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1559:   PetscCall(PetscObjectGetName((PetscObject)sectiondm, &sectiondm_name));
1560:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1561:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1562:   PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1563:   PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1564:   PetscCall(DMGetGlobalSection(sectiondm, &gsection));
1565:   /* Save raw section */
1566:   PetscCall(PetscSectionView(gsection, viewer));
1567:   /* Save plex wrapper */
1568:   {
1569:     PetscInt        pStart, pEnd, p, n;
1570:     IS              globalPointNumbers;
1571:     const PetscInt *gpoints;
1572:     IS              orderIS;
1573:     PetscInt       *order;

1575:     PetscCall(PetscSectionGetChart(gsection, &pStart, &pEnd));
1576:     PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbers));
1577:     PetscCall(ISGetIndices(globalPointNumbers, &gpoints));
1578:     for (p = pStart, n = 0; p < pEnd; ++p)
1579:       if (gpoints[p] >= 0) n++;
1580:     /* "order" is an array of global point numbers.
1581:        When loading, it is used with topology/order array
1582:        to match section points with plex topology points. */
1583:     PetscCall(PetscMalloc1(n, &order));
1584:     for (p = pStart, n = 0; p < pEnd; ++p)
1585:       if (gpoints[p] >= 0) order[n++] = gpoints[p];
1586:     PetscCall(ISRestoreIndices(globalPointNumbers, &gpoints));
1587:     PetscCall(ISDestroy(&globalPointNumbers));
1588:     PetscCall(ISCreateGeneral(comm, n, order, PETSC_OWN_POINTER, &orderIS));
1589:     PetscCall(PetscObjectSetName((PetscObject)orderIS, "order"));
1590:     PetscCall(ISView(orderIS, viewer));
1591:     PetscCall(ISDestroy(&orderIS));
1592:   }
1593:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1594:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1595:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1596:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1597:   PetscFunctionReturn(PETSC_SUCCESS);
1598: }

1600: PetscErrorCode DMPlexGlobalVectorView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1601: {
1602:   const char *topologydm_name;
1603:   const char *sectiondm_name;
1604:   const char *vec_name;
1605:   PetscInt    bs;

1607:   PetscFunctionBegin;
1608:   /* Check consistency */
1609:   {
1610:     PetscSF pointsf, pointsf1;

1612:     PetscCall(DMGetPointSF(dm, &pointsf));
1613:     PetscCall(DMGetPointSF(sectiondm, &pointsf1));
1614:     PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
1615:   }
1616:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1617:   PetscCall(PetscObjectGetName((PetscObject)sectiondm, &sectiondm_name));
1618:   PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
1619:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1620:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1621:   PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1622:   PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1623:   PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
1624:   PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
1625:   PetscCall(VecGetBlockSize(vec, &bs));
1626:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "blockSize", PETSC_INT, (void *)&bs));
1627:   PetscCall(VecSetBlockSize(vec, 1));
1628:   /* VecView(vec, viewer) would call (*vec->opt->view)(vec, viewer), but,    */
1629:   /* if vec was created with DMGet{Global, Local}Vector(), vec->opt->view    */
1630:   /* is set to VecView_Plex, which would save vec in a predefined location.  */
1631:   /* To save vec in where we want, we create a new Vec (temp) with           */
1632:   /* VecCreate(), wrap the vec data in temp, and call VecView(temp, viewer). */
1633:   {
1634:     Vec                temp;
1635:     const PetscScalar *array;
1636:     PetscLayout        map;

1638:     PetscCall(VecCreate(PetscObjectComm((PetscObject)vec), &temp));
1639:     PetscCall(PetscObjectSetName((PetscObject)temp, vec_name));
1640:     PetscCall(VecGetLayout(vec, &map));
1641:     PetscCall(VecSetLayout(temp, map));
1642:     PetscCall(VecSetUp(temp));
1643:     PetscCall(VecGetArrayRead(vec, &array));
1644:     PetscCall(VecPlaceArray(temp, array));
1645:     PetscCall(VecView(temp, viewer));
1646:     PetscCall(VecResetArray(temp));
1647:     PetscCall(VecRestoreArrayRead(vec, &array));
1648:     PetscCall(VecDestroy(&temp));
1649:   }
1650:   PetscCall(VecSetBlockSize(vec, bs));
1651:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1652:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1653:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1654:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1655:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1656:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1657:   PetscFunctionReturn(PETSC_SUCCESS);
1658: }

1660: PetscErrorCode DMPlexLocalVectorView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1661: {
1662:   MPI_Comm     comm;
1663:   const char  *topologydm_name;
1664:   const char  *sectiondm_name;
1665:   const char  *vec_name;
1666:   PetscSection section;
1667:   PetscBool    includesConstraints;
1668:   Vec          gvec;
1669:   PetscInt     m, bs;

1671:   PetscFunctionBegin;
1672:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1673:   /* Check consistency */
1674:   {
1675:     PetscSF pointsf, pointsf1;

1677:     PetscCall(DMGetPointSF(dm, &pointsf));
1678:     PetscCall(DMGetPointSF(sectiondm, &pointsf1));
1679:     PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
1680:   }
1681:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1682:   PetscCall(PetscObjectGetName((PetscObject)sectiondm, &sectiondm_name));
1683:   PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
1684:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1685:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1686:   PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1687:   PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1688:   PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
1689:   PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
1690:   PetscCall(VecGetBlockSize(vec, &bs));
1691:   PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "blockSize", PETSC_INT, (void *)&bs));
1692:   PetscCall(VecCreate(comm, &gvec));
1693:   PetscCall(PetscObjectSetName((PetscObject)gvec, vec_name));
1694:   PetscCall(DMGetGlobalSection(sectiondm, &section));
1695:   PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
1696:   if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
1697:   else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
1698:   PetscCall(VecSetSizes(gvec, m, PETSC_DECIDE));
1699:   PetscCall(VecSetUp(gvec));
1700:   PetscCall(DMLocalToGlobalBegin(sectiondm, vec, INSERT_VALUES, gvec));
1701:   PetscCall(DMLocalToGlobalEnd(sectiondm, vec, INSERT_VALUES, gvec));
1702:   PetscCall(VecView(gvec, viewer));
1703:   PetscCall(VecDestroy(&gvec));
1704:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1705:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1706:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1707:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1708:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1709:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1710:   PetscFunctionReturn(PETSC_SUCCESS);
1711: }

1713: struct _n_LoadLabelsCtx {
1714:   MPI_Comm    comm;
1715:   PetscMPIInt rank;
1716:   DM          dm;
1717:   PetscViewer viewer;
1718:   DMLabel     label;
1719:   PetscSF     sfXC;
1720:   PetscLayout layoutX;
1721: };
1722: typedef struct _n_LoadLabelsCtx *LoadLabelsCtx;

1724: static PetscErrorCode LoadLabelsCtxCreate(DM dm, PetscViewer viewer, PetscSF sfXC, LoadLabelsCtx *ctx)
1725: {
1726:   PetscFunctionBegin;
1727:   PetscCall(PetscNew(ctx));
1728:   PetscCall(PetscObjectReference((PetscObject)((*ctx)->dm = dm)));
1729:   PetscCall(PetscObjectReference((PetscObject)((*ctx)->viewer = viewer)));
1730:   PetscCall(PetscObjectGetComm((PetscObject)dm, &(*ctx)->comm));
1731:   PetscCallMPI(MPI_Comm_rank((*ctx)->comm, &(*ctx)->rank));
1732:   (*ctx)->sfXC = sfXC;
1733:   if (sfXC) {
1734:     PetscInt nX;

1736:     PetscCall(PetscObjectReference((PetscObject)sfXC));
1737:     PetscCall(PetscSFGetGraph(sfXC, &nX, NULL, NULL, NULL));
1738:     PetscCall(PetscLayoutCreateFromSizes((*ctx)->comm, nX, PETSC_DECIDE, 1, &(*ctx)->layoutX));
1739:   }
1740:   PetscFunctionReturn(PETSC_SUCCESS);
1741: }

1743: static PetscErrorCode LoadLabelsCtxDestroy(LoadLabelsCtx *ctx)
1744: {
1745:   PetscFunctionBegin;
1746:   if (!*ctx) PetscFunctionReturn(PETSC_SUCCESS);
1747:   PetscCall(DMDestroy(&(*ctx)->dm));
1748:   PetscCall(PetscViewerDestroy(&(*ctx)->viewer));
1749:   PetscCall(PetscSFDestroy(&(*ctx)->sfXC));
1750:   PetscCall(PetscLayoutDestroy(&(*ctx)->layoutX));
1751:   PetscCall(PetscFree(*ctx));
1752:   PetscFunctionReturn(PETSC_SUCCESS);
1753: }

1755: /*
1756:     A: on-disk points
1757:     X: global points [0, NX)
1758:     C: distributed plex points
1759: */
1760: static herr_t ReadLabelStratumHDF5_Distribute_Private(IS stratumIS, LoadLabelsCtx ctx, IS *newStratumIS)
1761: {
1762:   MPI_Comm        comm    = ctx->comm;
1763:   PetscSF         sfXC    = ctx->sfXC;
1764:   PetscLayout     layoutX = ctx->layoutX;
1765:   PetscSF         sfXA;
1766:   const PetscInt *A_points;
1767:   PetscInt        nX, nC;
1768:   PetscInt        n;

1770:   PetscFunctionBegin;
1771:   PetscCall(PetscSFGetGraph(sfXC, &nX, &nC, NULL, NULL));
1772:   PetscCall(ISGetLocalSize(stratumIS, &n));
1773:   PetscCall(ISGetIndices(stratumIS, &A_points));
1774:   PetscCall(PetscSFCreate(comm, &sfXA));
1775:   PetscCall(PetscSFSetGraphLayout(sfXA, layoutX, n, NULL, PETSC_USE_POINTER, A_points));
1776:   PetscCall(ISCreate(comm, newStratumIS));
1777:   PetscCall(ISSetType(*newStratumIS, ISGENERAL));
1778:   {
1779:     PetscInt   i;
1780:     PetscBool *A_mask, *X_mask, *C_mask;

1782:     PetscCall(PetscCalloc3(n, &A_mask, nX, &X_mask, nC, &C_mask));
1783:     for (i = 0; i < n; i++) A_mask[i] = PETSC_TRUE;
1784:     PetscCall(PetscSFReduceBegin(sfXA, MPIU_BOOL, A_mask, X_mask, MPI_REPLACE));
1785:     PetscCall(PetscSFReduceEnd(sfXA, MPIU_BOOL, A_mask, X_mask, MPI_REPLACE));
1786:     PetscCall(PetscSFBcastBegin(sfXC, MPIU_BOOL, X_mask, C_mask, MPI_LOR));
1787:     PetscCall(PetscSFBcastEnd(sfXC, MPIU_BOOL, X_mask, C_mask, MPI_LOR));
1788:     PetscCall(ISGeneralSetIndicesFromMask(*newStratumIS, 0, nC, C_mask));
1789:     PetscCall(PetscFree3(A_mask, X_mask, C_mask));
1790:   }
1791:   PetscCall(PetscSFDestroy(&sfXA));
1792:   PetscCall(ISRestoreIndices(stratumIS, &A_points));
1793:   PetscFunctionReturn(PETSC_SUCCESS);
1794: }

1796: static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *vname, const H5L_info_t *info, void *op_data)
1797: {
1798:   LoadLabelsCtx   ctx    = (LoadLabelsCtx)op_data;
1799:   PetscViewer     viewer = ctx->viewer;
1800:   DMLabel         label  = ctx->label;
1801:   MPI_Comm        comm   = ctx->comm;
1802:   IS              stratumIS;
1803:   const PetscInt *ind;
1804:   PetscInt        value, N, i;

1806:   PetscCall(PetscOptionsStringToInt(vname, &value));
1807:   PetscCall(ISCreate(comm, &stratumIS));
1808:   PetscCall(PetscObjectSetName((PetscObject)stratumIS, "indices"));
1809:   PetscCall(PetscViewerHDF5PushGroup(viewer, vname)); /* labels/<lname>/<vname> */

1811:   if (!ctx->sfXC) {
1812:     /* Force serial load */
1813:     PetscCall(PetscViewerHDF5ReadSizes(viewer, "indices", NULL, &N));
1814:     PetscCall(PetscLayoutSetLocalSize(stratumIS->map, !ctx->rank ? N : 0));
1815:     PetscCall(PetscLayoutSetSize(stratumIS->map, N));
1816:   }
1817:   PetscCall(ISLoad(stratumIS, viewer));

1819:   if (ctx->sfXC) {
1820:     IS newStratumIS;

1822:     PetscCallHDF5(ReadLabelStratumHDF5_Distribute_Private, (stratumIS, ctx, &newStratumIS));
1823:     PetscCall(ISDestroy(&stratumIS));
1824:     stratumIS = newStratumIS;
1825:   }

1827:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1828:   PetscCall(ISGetLocalSize(stratumIS, &N));
1829:   PetscCall(ISGetIndices(stratumIS, &ind));
1830:   for (i = 0; i < N; ++i) PetscCall(DMLabelSetValue(label, ind[i], value));
1831:   PetscCall(ISRestoreIndices(stratumIS, &ind));
1832:   PetscCall(ISDestroy(&stratumIS));
1833:   return 0;
1834: }

1836: /* TODO: Fix this code, it is returning PETSc error codes when it should be translating them to herr_t codes */
1837: static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *lname, const H5L_info_t *info, void *op_data)
1838: {
1839:   LoadLabelsCtx  ctx = (LoadLabelsCtx)op_data;
1840:   DM             dm  = ctx->dm;
1841:   hsize_t        idx = 0;
1842:   PetscErrorCode ierr;
1843:   PetscBool      flg;
1844:   herr_t         err;

1846:   PetscCall(DMHasLabel(dm, lname, &flg));
1847:   if (flg) PetscCall(DMRemoveLabel(dm, lname, NULL));
1848:   ierr = DMCreateLabel(dm, lname);
1849:   if (ierr) return (herr_t)ierr;
1850:   ierr = DMGetLabel(dm, lname, &ctx->label);
1851:   if (ierr) return (herr_t)ierr;
1852:   ierr = PetscViewerHDF5PushGroup(ctx->viewer, lname);
1853:   if (ierr) return (herr_t)ierr;
1854:   /* Iterate over the label's strata */
1855:   PetscCallHDF5Return(err, H5Literate_by_name, (g_id, lname, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0));
1856:   ierr = PetscViewerHDF5PopGroup(ctx->viewer);
1857:   if (ierr) return (herr_t)ierr;
1858:   return err;
1859: }

1861: PetscErrorCode DMPlexLabelsLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF sfXC)
1862: {
1863:   const char          *topologydm_name;
1864:   LoadLabelsCtx        ctx;
1865:   hsize_t              idx = 0;
1866:   char                 group[PETSC_MAX_PATH_LEN];
1867:   DMPlexStorageVersion version;
1868:   PetscBool            distributed, hasGroup;

1870:   PetscFunctionBegin;
1871:   PetscCall(DMPlexIsDistributed(dm, &distributed));
1872:   if (distributed) PetscCheck(sfXC, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_NULL, "PetscSF must be given for parallel load");
1873:   PetscCall(LoadLabelsCtxCreate(dm, viewer, sfXC, &ctx));
1874:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1875:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
1876:   if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1877:     PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s/labels", topologydm_name));
1878:   } else {
1879:     PetscCall(PetscStrncpy(group, "labels", sizeof(group)));
1880:   }
1881:   PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1882:   PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &hasGroup));
1883:   if (hasGroup) {
1884:     hid_t fileId, groupId;

1886:     PetscCall(PetscViewerHDF5OpenGroup(viewer, NULL, &fileId, &groupId));
1887:     /* Iterate over labels */
1888:     PetscCallHDF5(H5Literate, (groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, ctx));
1889:     PetscCallHDF5(H5Gclose, (groupId));
1890:   }
1891:   PetscCall(PetscViewerHDF5PopGroup(viewer));
1892:   PetscCall(LoadLabelsCtxDestroy(&ctx));
1893:   PetscFunctionReturn(PETSC_SUCCESS);
1894: }

1896: static PetscErrorCode DMPlexDistributionLoad_HDF5_Private(DM dm, PetscViewer viewer, PetscSF sf, PetscSF *distsf, DM *distdm)
1897: {
1898:   MPI_Comm        comm;
1899:   PetscMPIInt     size, rank;
1900:   PetscInt        dist_size;
1901:   const char     *distribution_name;
1902:   PetscInt        p, lsize;
1903:   IS              chartSizesIS, ownersIS, gpointsIS;
1904:   const PetscInt *chartSize, *owners, *gpoints;
1905:   PetscLayout     layout;
1906:   PetscBool       has;

1908:   PetscFunctionBegin;
1909:   *distsf = NULL;
1910:   *distdm = NULL;
1911:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1912:   PetscCallMPI(MPI_Comm_size(comm, &size));
1913:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
1914:   PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
1915:   if (!distribution_name) PetscFunctionReturn(PETSC_SUCCESS);
1916:   PetscCall(PetscLogEventBegin(DMPLEX_DistributionLoad, viewer, 0, 0, 0));
1917:   PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &has));
1918:   if (!has) {
1919:     char *full_group;

1921:     PetscCall(PetscViewerHDF5GetGroup(viewer, NULL, &full_group));
1922:     PetscCheck(has, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Distribution %s cannot be found: HDF5 group %s not found in file", distribution_name, full_group);
1923:   }
1924:   PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "comm_size", PETSC_INT, NULL, (void *)&dist_size));
1925:   PetscCheck(dist_size == (PetscInt)size, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Mismatching comm sizes: comm size of this session (%d) != comm size used for %s (%" PetscInt_FMT ")", size, distribution_name, dist_size);
1926:   PetscCall(ISCreate(comm, &chartSizesIS));
1927:   PetscCall(PetscObjectSetName((PetscObject)chartSizesIS, "chart_sizes"));
1928:   PetscCall(ISCreate(comm, &ownersIS));
1929:   PetscCall(PetscObjectSetName((PetscObject)ownersIS, "owners"));
1930:   PetscCall(ISCreate(comm, &gpointsIS));
1931:   PetscCall(PetscObjectSetName((PetscObject)gpointsIS, "global_point_numbers"));
1932:   PetscCall(PetscLayoutSetLocalSize(chartSizesIS->map, 1));
1933:   PetscCall(ISLoad(chartSizesIS, viewer));
1934:   PetscCall(ISGetIndices(chartSizesIS, &chartSize));
1935:   PetscCall(PetscLayoutSetLocalSize(ownersIS->map, *chartSize));
1936:   PetscCall(PetscLayoutSetLocalSize(gpointsIS->map, *chartSize));
1937:   PetscCall(ISLoad(ownersIS, viewer));
1938:   PetscCall(ISLoad(gpointsIS, viewer));
1939:   PetscCall(ISGetIndices(ownersIS, &owners));
1940:   PetscCall(ISGetIndices(gpointsIS, &gpoints));
1941:   PetscCall(PetscSFCreate(comm, distsf));
1942:   PetscCall(PetscSFSetFromOptions(*distsf));
1943:   PetscCall(PetscLayoutCreate(comm, &layout));
1944:   PetscCall(PetscSFGetGraph(sf, &lsize, NULL, NULL, NULL));
1945:   PetscCall(PetscLayoutSetLocalSize(layout, lsize));
1946:   PetscCall(PetscLayoutSetBlockSize(layout, 1));
1947:   PetscCall(PetscLayoutSetUp(layout));
1948:   PetscCall(PetscSFSetGraphLayout(*distsf, layout, *chartSize, NULL, PETSC_OWN_POINTER, gpoints));
1949:   PetscCall(PetscLayoutDestroy(&layout));
1950:   /* Migrate DM */
1951:   {
1952:     PetscInt     pStart, pEnd;
1953:     PetscSFNode *buffer0, *buffer1, *buffer2;

1955:     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1956:     PetscCall(PetscMalloc2(pEnd - pStart, &buffer0, lsize, &buffer1));
1957:     PetscCall(PetscMalloc1(*chartSize, &buffer2));
1958:     {
1959:       PetscSF            workPointSF;
1960:       PetscInt           workNroots, workNleaves;
1961:       const PetscInt    *workIlocal;
1962:       const PetscSFNode *workIremote;

1964:       for (p = pStart; p < pEnd; ++p) {
1965:         buffer0[p - pStart].rank  = rank;
1966:         buffer0[p - pStart].index = p - pStart;
1967:       }
1968:       PetscCall(DMGetPointSF(dm, &workPointSF));
1969:       PetscCall(PetscSFGetGraph(workPointSF, &workNroots, &workNleaves, &workIlocal, &workIremote));
1970:       for (p = 0; p < workNleaves; ++p) {
1971:         PetscInt workIlocalp = (workIlocal ? workIlocal[p] : p);

1973:         buffer0[workIlocalp].rank = -1;
1974:       }
1975:     }
1976:     for (p = 0; p < lsize; ++p) buffer1[p].rank = -1;
1977:     for (p = 0; p < *chartSize; ++p) buffer2[p].rank = -1;
1978:     PetscCall(PetscSFReduceBegin(sf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
1979:     PetscCall(PetscSFReduceEnd(sf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
1980:     PetscCall(PetscSFBcastBegin(*distsf, MPIU_SF_NODE, buffer1, buffer2, MPI_REPLACE));
1981:     PetscCall(PetscSFBcastEnd(*distsf, MPIU_SF_NODE, buffer1, buffer2, MPI_REPLACE));
1982:     if (PetscDefined(USE_DEBUG)) {
1983:       for (p = 0; p < *chartSize; ++p) {
1984:         PetscCheck(buffer2[p].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Found negative root rank %" PetscInt_FMT " at local point %" PetscInt_FMT " on rank %d when making migrationSF", buffer2[p].rank, p, rank);
1985:       }
1986:     }
1987:     PetscCall(PetscFree2(buffer0, buffer1));
1988:     PetscCall(DMCreate(comm, distdm));
1989:     PetscCall(DMSetType(*distdm, DMPLEX));
1990:     PetscCall(PetscObjectSetName((PetscObject)*distdm, ((PetscObject)dm)->name));
1991:     PetscCall(DMPlexDistributionSetName(*distdm, distribution_name));
1992:     {
1993:       PetscSF migrationSF;

1995:       PetscCall(PetscSFCreate(comm, &migrationSF));
1996:       PetscCall(PetscSFSetFromOptions(migrationSF));
1997:       PetscCall(PetscSFSetGraph(migrationSF, pEnd - pStart, *chartSize, NULL, PETSC_OWN_POINTER, buffer2, PETSC_OWN_POINTER));
1998:       PetscCall(PetscSFSetUp(migrationSF));
1999:       PetscCall(DMPlexMigrate(dm, migrationSF, *distdm));
2000:       PetscCall(PetscSFDestroy(&migrationSF));
2001:     }
2002:   }
2003:   /* Set pointSF */
2004:   {
2005:     PetscSF      pointSF;
2006:     PetscInt    *ilocal, nleaves, q;
2007:     PetscSFNode *iremote, *buffer0, *buffer1;

2009:     PetscCall(PetscMalloc2(*chartSize, &buffer0, lsize, &buffer1));
2010:     for (p = 0, nleaves = 0; p < *chartSize; ++p) {
2011:       if (owners[p] == rank) {
2012:         buffer0[p].rank = rank;
2013:       } else {
2014:         buffer0[p].rank = -1;
2015:         nleaves++;
2016:       }
2017:       buffer0[p].index = p;
2018:     }
2019:     for (p = 0; p < lsize; ++p) buffer1[p].rank = -1;
2020:     PetscCall(PetscSFReduceBegin(*distsf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
2021:     PetscCall(PetscSFReduceEnd(*distsf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
2022:     for (p = 0; p < *chartSize; ++p) buffer0[p].rank = -1;
2023:     PetscCall(PetscSFBcastBegin(*distsf, MPIU_SF_NODE, buffer1, buffer0, MPI_REPLACE));
2024:     PetscCall(PetscSFBcastEnd(*distsf, MPIU_SF_NODE, buffer1, buffer0, MPI_REPLACE));
2025:     if (PetscDefined(USE_DEBUG)) {
2026:       for (p = 0; p < *chartSize; ++p) PetscCheck(buffer0[p].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Found negative root rank %" PetscInt_FMT " at local point %" PetscInt_FMT " on rank %d when making pointSF", buffer0[p].rank, p, rank);
2027:     }
2028:     PetscCall(PetscMalloc1(nleaves, &ilocal));
2029:     PetscCall(PetscMalloc1(nleaves, &iremote));
2030:     for (p = 0, q = 0; p < *chartSize; ++p) {
2031:       if (buffer0[p].rank != rank) {
2032:         ilocal[q]        = p;
2033:         iremote[q].rank  = buffer0[p].rank;
2034:         iremote[q].index = buffer0[p].index;
2035:         q++;
2036:       }
2037:     }
2038:     PetscCheck(q == nleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching leaf sizes: %" PetscInt_FMT " != %" PetscInt_FMT, q, nleaves);
2039:     PetscCall(PetscFree2(buffer0, buffer1));
2040:     PetscCall(PetscSFCreate(comm, &pointSF));
2041:     PetscCall(PetscSFSetFromOptions(pointSF));
2042:     PetscCall(PetscSFSetGraph(pointSF, *chartSize, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
2043:     PetscCall(DMSetPointSF(*distdm, pointSF));
2044:     {
2045:       DM cdm;

2047:       PetscCall(DMGetCoordinateDM(*distdm, &cdm));
2048:       PetscCall(DMSetPointSF(cdm, pointSF));
2049:     }
2050:     PetscCall(PetscSFDestroy(&pointSF));
2051:   }
2052:   PetscCall(ISRestoreIndices(chartSizesIS, &chartSize));
2053:   PetscCall(ISRestoreIndices(ownersIS, &owners));
2054:   PetscCall(ISRestoreIndices(gpointsIS, &gpoints));
2055:   PetscCall(ISDestroy(&chartSizesIS));
2056:   PetscCall(ISDestroy(&ownersIS));
2057:   PetscCall(ISDestroy(&gpointsIS));
2058:   /* Record that overlap has been manually created.               */
2059:   /* This is to pass `DMPlexCheckPointSF()`, which checks that    */
2060:   /* pointSF does not contain cells in the leaves if overlap = 0. */
2061:   PetscCall(DMPlexSetOverlap_Plex(*distdm, NULL, DMPLEX_OVERLAP_MANUAL));
2062:   PetscCall(DMPlexDistributeSetDefault(*distdm, PETSC_FALSE));
2063:   PetscCall(DMPlexReorderSetDefault(*distdm, DM_REORDER_DEFAULT_FALSE));
2064:   PetscCall(PetscLogEventEnd(DMPLEX_DistributionLoad, viewer, 0, 0, 0));
2065:   PetscFunctionReturn(PETSC_SUCCESS);
2066: }

2068: // Serial load of topology
2069: static PetscErrorCode DMPlexTopologyLoad_HDF5_Legacy_Private(DM dm, PetscViewer viewer, PetscSF *sf)
2070: {
2071:   MPI_Comm        comm;
2072:   const char     *pointsName, *coneSizesName, *conesName, *orientationsName;
2073:   IS              pointsIS, coneSizesIS, conesIS, orientationsIS;
2074:   const PetscInt *points, *coneSizes, *cones, *orientations;
2075:   PetscInt       *cone, *ornt;
2076:   PetscInt        dim, N, Np, pEnd, p, q, maxConeSize = 0, c;
2077:   PetscMPIInt     size, rank;

2079:   PetscFunctionBegin;
2080:   pointsName       = "order";
2081:   coneSizesName    = "cones";
2082:   conesName        = "cells";
2083:   orientationsName = "orientation";
2084:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2085:   PetscCallMPI(MPI_Comm_size(comm, &size));
2086:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
2087:   PetscCall(ISCreate(comm, &pointsIS));
2088:   PetscCall(PetscObjectSetName((PetscObject)pointsIS, pointsName));
2089:   PetscCall(ISCreate(comm, &coneSizesIS));
2090:   PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
2091:   PetscCall(ISCreate(comm, &conesIS));
2092:   PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
2093:   PetscCall(ISCreate(comm, &orientationsIS));
2094:   PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
2095:   PetscCall(PetscViewerHDF5ReadObjectAttribute(viewer, (PetscObject)conesIS, "cell_dim", PETSC_INT, NULL, &dim));
2096:   PetscCall(DMSetDimension(dm, dim));
2097:   {
2098:     /* Force serial load */
2099:     PetscCall(PetscInfo(dm, "Loading DM %s in serial\n", dm->hdr.name));
2100:     PetscCall(PetscViewerHDF5ReadSizes(viewer, pointsName, NULL, &Np));
2101:     PetscCall(PetscLayoutSetLocalSize(pointsIS->map, rank == 0 ? Np : 0));
2102:     PetscCall(PetscLayoutSetSize(pointsIS->map, Np));
2103:     pEnd = rank == 0 ? Np : 0;
2104:     PetscCall(PetscViewerHDF5ReadSizes(viewer, coneSizesName, NULL, &Np));
2105:     PetscCall(PetscLayoutSetLocalSize(coneSizesIS->map, rank == 0 ? Np : 0));
2106:     PetscCall(PetscLayoutSetSize(coneSizesIS->map, Np));
2107:     PetscCall(PetscViewerHDF5ReadSizes(viewer, conesName, NULL, &N));
2108:     PetscCall(PetscLayoutSetLocalSize(conesIS->map, rank == 0 ? N : 0));
2109:     PetscCall(PetscLayoutSetSize(conesIS->map, N));
2110:     PetscCall(PetscViewerHDF5ReadSizes(viewer, orientationsName, NULL, &N));
2111:     PetscCall(PetscLayoutSetLocalSize(orientationsIS->map, rank == 0 ? N : 0));
2112:     PetscCall(PetscLayoutSetSize(orientationsIS->map, N));
2113:   }
2114:   PetscCall(ISLoad(pointsIS, viewer));
2115:   PetscCall(ISLoad(coneSizesIS, viewer));
2116:   PetscCall(ISLoad(conesIS, viewer));
2117:   PetscCall(ISLoad(orientationsIS, viewer));
2118:   /* Create Plex */
2119:   PetscCall(DMPlexSetChart(dm, 0, pEnd));
2120:   PetscCall(ISGetIndices(pointsIS, &points));
2121:   PetscCall(ISGetIndices(coneSizesIS, &coneSizes));
2122:   for (p = 0; p < pEnd; ++p) {
2123:     PetscCall(DMPlexSetConeSize(dm, points[p], coneSizes[p]));
2124:     maxConeSize = PetscMax(maxConeSize, coneSizes[p]);
2125:   }
2126:   PetscCall(DMSetUp(dm));
2127:   PetscCall(ISGetIndices(conesIS, &cones));
2128:   PetscCall(ISGetIndices(orientationsIS, &orientations));
2129:   PetscCall(PetscMalloc2(maxConeSize, &cone, maxConeSize, &ornt));
2130:   for (p = 0, q = 0; p < pEnd; ++p) {
2131:     for (c = 0; c < coneSizes[p]; ++c, ++q) {
2132:       cone[c] = cones[q];
2133:       ornt[c] = orientations[q];
2134:     }
2135:     PetscCall(DMPlexSetCone(dm, points[p], cone));
2136:     PetscCall(DMPlexSetConeOrientation(dm, points[p], ornt));
2137:   }
2138:   PetscCall(PetscFree2(cone, ornt));
2139:   /* Create global section migration SF */
2140:   if (sf) {
2141:     PetscLayout layout;
2142:     PetscInt   *globalIndices;

2144:     PetscCall(PetscMalloc1(pEnd, &globalIndices));
2145:     /* plex point == globalPointNumber in this case */
2146:     for (p = 0; p < pEnd; ++p) globalIndices[p] = p;
2147:     PetscCall(PetscLayoutCreate(comm, &layout));
2148:     PetscCall(PetscLayoutSetSize(layout, Np));
2149:     PetscCall(PetscLayoutSetBlockSize(layout, 1));
2150:     PetscCall(PetscLayoutSetUp(layout));
2151:     PetscCall(PetscSFCreate(comm, sf));
2152:     PetscCall(PetscSFSetFromOptions(*sf));
2153:     PetscCall(PetscSFSetGraphLayout(*sf, layout, pEnd, NULL, PETSC_OWN_POINTER, globalIndices));
2154:     PetscCall(PetscLayoutDestroy(&layout));
2155:     PetscCall(PetscFree(globalIndices));
2156:   }
2157:   /* Clean-up */
2158:   PetscCall(ISRestoreIndices(pointsIS, &points));
2159:   PetscCall(ISRestoreIndices(coneSizesIS, &coneSizes));
2160:   PetscCall(ISRestoreIndices(conesIS, &cones));
2161:   PetscCall(ISRestoreIndices(orientationsIS, &orientations));
2162:   PetscCall(ISDestroy(&pointsIS));
2163:   PetscCall(ISDestroy(&coneSizesIS));
2164:   PetscCall(ISDestroy(&conesIS));
2165:   PetscCall(ISDestroy(&orientationsIS));
2166:   /* Fill in the rest of the topology structure */
2167:   PetscCall(DMPlexSymmetrize(dm));
2168:   PetscCall(DMPlexStratify(dm));
2169:   PetscFunctionReturn(PETSC_SUCCESS);
2170: }

2172: /* Representation of two DMPlex strata in 0-based global numbering */
2173: struct _n_PlexLayer {
2174:   PetscInt     d;
2175:   IS           conesIS, orientationsIS;
2176:   PetscSection coneSizesSection;
2177:   PetscLayout  vertexLayout;
2178:   PetscSF      overlapSF, l2gSF; //TODO maybe confusing names (in DMPlex in general)
2179:   PetscInt     offset, conesOffset, leafOffset;
2180: };
2181: typedef struct _n_PlexLayer *PlexLayer;

2183: static PetscErrorCode PlexLayerDestroy(PlexLayer *layer)
2184: {
2185:   PetscFunctionBegin;
2186:   if (!*layer) PetscFunctionReturn(PETSC_SUCCESS);
2187:   PetscCall(PetscSectionDestroy(&(*layer)->coneSizesSection));
2188:   PetscCall(ISDestroy(&(*layer)->conesIS));
2189:   PetscCall(ISDestroy(&(*layer)->orientationsIS));
2190:   PetscCall(PetscSFDestroy(&(*layer)->overlapSF));
2191:   PetscCall(PetscSFDestroy(&(*layer)->l2gSF));
2192:   PetscCall(PetscLayoutDestroy(&(*layer)->vertexLayout));
2193:   PetscCall(PetscFree(*layer));
2194:   PetscFunctionReturn(PETSC_SUCCESS);
2195: }

2197: static PetscErrorCode PlexLayerCreate_Private(PlexLayer *layer)
2198: {
2199:   PetscFunctionBegin;
2200:   PetscCall(PetscNew(layer));
2201:   (*layer)->d           = -1;
2202:   (*layer)->offset      = -1;
2203:   (*layer)->conesOffset = -1;
2204:   (*layer)->leafOffset  = -1;
2205:   PetscFunctionReturn(PETSC_SUCCESS);
2206: }

2208: // Parallel load of a depth stratum
2209: static PetscErrorCode PlexLayerLoad_Private(PlexLayer layer, PetscViewer viewer, PetscInt d, PetscLayout pointsLayout)
2210: {
2211:   char         path[128];
2212:   MPI_Comm     comm;
2213:   const char  *coneSizesName, *conesName, *orientationsName;
2214:   IS           coneSizesIS, conesIS, orientationsIS;
2215:   PetscSection coneSizesSection;
2216:   PetscLayout  vertexLayout = NULL;
2217:   PetscInt     s;

2219:   PetscFunctionBegin;
2220:   coneSizesName    = "cone_sizes";
2221:   conesName        = "cones";
2222:   orientationsName = "orientations";
2223:   PetscCall(PetscObjectGetComm((PetscObject)viewer, &comm));

2225:   /* query size of next lower depth stratum (next lower dimension) */
2226:   if (d > 0) {
2227:     PetscInt NVertices;

2229:     PetscCall(PetscSNPrintf(path, sizeof(path), "%" PetscInt_FMT "/%s", d - 1, coneSizesName));
2230:     PetscCall(PetscViewerHDF5ReadSizes(viewer, path, NULL, &NVertices));
2231:     PetscCall(PetscLayoutCreate(comm, &vertexLayout));
2232:     PetscCall(PetscLayoutSetSize(vertexLayout, NVertices));
2233:     PetscCall(PetscLayoutSetUp(vertexLayout));
2234:   }

2236:   PetscCall(PetscSNPrintf(path, sizeof(path), "%" PetscInt_FMT, d));
2237:   PetscCall(PetscViewerHDF5PushGroup(viewer, path));

2239:   /* create coneSizesSection from stored IS coneSizes */
2240:   {
2241:     const PetscInt *coneSizes;

2243:     PetscCall(ISCreate(comm, &coneSizesIS));
2244:     PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
2245:     if (pointsLayout) PetscCall(ISSetLayout(coneSizesIS, pointsLayout));
2246:     PetscCall(ISLoad(coneSizesIS, viewer));
2247:     if (!pointsLayout) PetscCall(ISGetLayout(coneSizesIS, &pointsLayout));
2248:     PetscCall(ISGetIndices(coneSizesIS, &coneSizes));
2249:     PetscCall(PetscSectionCreate(comm, &coneSizesSection));
2250:     //TODO different start ?
2251:     PetscCall(PetscSectionSetChart(coneSizesSection, 0, pointsLayout->n));
2252:     for (s = 0; s < pointsLayout->n; ++s) PetscCall(PetscSectionSetDof(coneSizesSection, s, coneSizes[s]));
2253:     PetscCall(PetscSectionSetUp(coneSizesSection));
2254:     PetscCall(ISRestoreIndices(coneSizesIS, &coneSizes));
2255:     {
2256:       PetscLayout tmp = NULL;

2258:       /* We need to keep the layout until the end of function */
2259:       PetscCall(PetscLayoutReference((PetscLayout)pointsLayout, &tmp));
2260:     }
2261:     PetscCall(ISDestroy(&coneSizesIS));
2262:   }

2264:   /* use value layout of coneSizesSection as layout of cones and orientations */
2265:   {
2266:     PetscLayout conesLayout;

2268:     PetscCall(PetscSectionGetValueLayout(comm, coneSizesSection, &conesLayout));
2269:     PetscCall(ISCreate(comm, &conesIS));
2270:     PetscCall(ISCreate(comm, &orientationsIS));
2271:     PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
2272:     PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
2273:     PetscCall(PetscLayoutDuplicate(conesLayout, &conesIS->map));
2274:     PetscCall(PetscLayoutDuplicate(conesLayout, &orientationsIS->map));
2275:     PetscCall(ISLoad(conesIS, viewer));
2276:     PetscCall(ISLoad(orientationsIS, viewer));
2277:     PetscCall(PetscLayoutDestroy(&conesLayout));
2278:   }

2280:   /* check assertion that layout of points is the same as point layout of coneSizesSection */
2281:   {
2282:     PetscLayout pointsLayout0;
2283:     PetscBool   flg;

2285:     PetscCall(PetscSectionGetPointLayout(comm, coneSizesSection, &pointsLayout0));
2286:     PetscCall(PetscLayoutCompare(pointsLayout, pointsLayout0, &flg));
2287:     PetscCheck(flg, comm, PETSC_ERR_PLIB, "points layout != coneSizesSection point layout");
2288:     PetscCall(PetscLayoutDestroy(&pointsLayout0));
2289:   }
2290:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2291:   PetscCall(PetscLayoutDestroy(&pointsLayout));

2293:   layer->d                = d;
2294:   layer->conesIS          = conesIS;
2295:   layer->coneSizesSection = coneSizesSection;
2296:   layer->orientationsIS   = orientationsIS;
2297:   layer->vertexLayout     = vertexLayout;
2298:   PetscFunctionReturn(PETSC_SUCCESS);
2299: }

2301: static PetscErrorCode PlexLayerDistribute_Private(PlexLayer layer, PetscSF cellLocalToGlobalSF)
2302: {
2303:   IS           newConesIS, newOrientationsIS;
2304:   PetscSection newConeSizesSection;
2305:   MPI_Comm     comm;

2307:   PetscFunctionBegin;
2308:   PetscCall(PetscObjectGetComm((PetscObject)cellLocalToGlobalSF, &comm));
2309:   PetscCall(PetscSectionCreate(comm, &newConeSizesSection));
2310:   //TODO rename to something like ISDistribute() with optional PetscSection argument
2311:   PetscCall(DMPlexDistributeFieldIS(NULL, cellLocalToGlobalSF, layer->coneSizesSection, layer->conesIS, newConeSizesSection, &newConesIS));
2312:   PetscCall(DMPlexDistributeFieldIS(NULL, cellLocalToGlobalSF, layer->coneSizesSection, layer->orientationsIS, newConeSizesSection, &newOrientationsIS));

2314:   PetscCall(PetscObjectSetName((PetscObject)newConeSizesSection, ((PetscObject)layer->coneSizesSection)->name));
2315:   PetscCall(PetscObjectSetName((PetscObject)newConesIS, ((PetscObject)layer->conesIS)->name));
2316:   PetscCall(PetscObjectSetName((PetscObject)newOrientationsIS, ((PetscObject)layer->orientationsIS)->name));
2317:   PetscCall(PetscSectionDestroy(&layer->coneSizesSection));
2318:   PetscCall(ISDestroy(&layer->conesIS));
2319:   PetscCall(ISDestroy(&layer->orientationsIS));
2320:   layer->coneSizesSection = newConeSizesSection;
2321:   layer->conesIS          = newConesIS;
2322:   layer->orientationsIS   = newOrientationsIS;
2323:   PetscFunctionReturn(PETSC_SUCCESS);
2324: }

2326:   //TODO share code with DMPlexBuildFromCellListParallel()
2327: #include <petsc/private/hashseti.h>
2328: static PetscErrorCode PlexLayerCreateSFs_Private(PlexLayer layer, PetscSF *vertexOverlapSF, PetscSF *sfXC)
2329: {
2330:   PetscLayout     vertexLayout     = layer->vertexLayout;
2331:   PetscSection    coneSection      = layer->coneSizesSection;
2332:   IS              cellVertexData   = layer->conesIS;
2333:   IS              coneOrientations = layer->orientationsIS;
2334:   PetscSF         vl2gSF, vOverlapSF;
2335:   PetscInt       *verticesAdj;
2336:   PetscInt        i, n, numVerticesAdj;
2337:   const PetscInt *cvd, *co = NULL;
2338:   MPI_Comm        comm;

2340:   PetscFunctionBegin;
2341:   PetscCall(PetscObjectGetComm((PetscObject)coneSection, &comm));
2342:   PetscCall(PetscSectionGetStorageSize(coneSection, &n));
2343:   {
2344:     PetscInt n0;

2346:     PetscCall(ISGetLocalSize(cellVertexData, &n0));
2347:     PetscCheck(n == n0, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Local size of IS cellVertexData = %" PetscInt_FMT " != %" PetscInt_FMT " = storage size of PetscSection coneSection", n0, n);
2348:     PetscCall(ISGetIndices(cellVertexData, &cvd));
2349:   }
2350:   if (coneOrientations) {
2351:     PetscInt n0;

2353:     PetscCall(ISGetLocalSize(coneOrientations, &n0));
2354:     PetscCheck(n == n0, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Local size of IS coneOrientations = %" PetscInt_FMT " != %" PetscInt_FMT " = storage size of PetscSection coneSection", n0, n);
2355:     PetscCall(ISGetIndices(coneOrientations, &co));
2356:   }
2357:   /* Get/check global number of vertices */
2358:   {
2359:     PetscInt NVerticesInCells = PETSC_INT_MIN;

2361:     /* NVerticesInCells = max(cellVertexData) + 1 */
2362:     for (i = 0; i < n; i++)
2363:       if (cvd[i] > NVerticesInCells) NVerticesInCells = cvd[i];
2364:     ++NVerticesInCells;
2365:     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, comm));

2367:     if (vertexLayout->n == PETSC_DECIDE && vertexLayout->N == PETSC_DECIDE) vertexLayout->N = NVerticesInCells;
2368:     else
2369:       PetscCheck(vertexLayout->N == PETSC_DECIDE || vertexLayout->N >= NVerticesInCells, comm, PETSC_ERR_ARG_SIZ, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of unique vertices in the cell-vertex dataset %" PetscInt_FMT,
2370:                  vertexLayout->N, NVerticesInCells);
2371:     PetscCall(PetscLayoutSetUp(vertexLayout));
2372:   }
2373:   /* Find locally unique vertices in cellVertexData */
2374:   {
2375:     PetscHSetI vhash;
2376:     PetscInt   off = 0;

2378:     PetscCall(PetscHSetICreate(&vhash));
2379:     for (i = 0; i < n; ++i) PetscCall(PetscHSetIAdd(vhash, cvd[i]));
2380:     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
2381:     PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
2382:     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
2383:     PetscAssert(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Assertion failed: off == numVerticesAdj (%" PetscInt_FMT " != %" PetscInt_FMT ")", off, numVerticesAdj);
2384:     PetscCall(PetscHSetIDestroy(&vhash));
2385:   }
2386:   /* We must sort vertices to preserve numbering */
2387:   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
2388:   /* Connect locally unique vertices with star forest */
2389:   PetscCall(PetscSFCreateByMatchingIndices(vertexLayout, numVerticesAdj, verticesAdj, NULL, 0, numVerticesAdj, verticesAdj, NULL, 0, &vl2gSF, &vOverlapSF));
2390:   PetscCall(PetscObjectSetName((PetscObject)vOverlapSF, "overlapSF"));
2391:   PetscCall(PetscObjectSetName((PetscObject)vl2gSF, "localToGlobalSF"));

2393:   PetscCall(PetscFree(verticesAdj));
2394:   *vertexOverlapSF = vOverlapSF;
2395:   *sfXC            = vl2gSF;
2396:   PetscFunctionReturn(PETSC_SUCCESS);
2397: }

2399: static PetscErrorCode PlexLayerCreateCellSFs_Private(PlexLayer layer, PetscSF *cellOverlapSF, PetscSF *cellLocalToGlobalSF)
2400: {
2401:   PetscSection coneSection = layer->coneSizesSection;
2402:   PetscInt     nCells;
2403:   MPI_Comm     comm;

2405:   PetscFunctionBegin;
2406:   PetscCall(PetscObjectGetComm((PetscObject)coneSection, &comm));
2407:   {
2408:     PetscInt cStart;

2410:     PetscCall(PetscSectionGetChart(coneSection, &cStart, &nCells));
2411:     PetscCheck(cStart == 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "coneSection must start at 0");
2412:   }
2413:   /* Create overlapSF as empty SF with the right number of roots */
2414:   PetscCall(PetscSFCreate(comm, cellOverlapSF));
2415:   PetscCall(PetscSFSetGraph(*cellOverlapSF, nCells, 0, NULL, PETSC_USE_POINTER, NULL, PETSC_USE_POINTER));
2416:   PetscCall(PetscSFSetUp(*cellOverlapSF));
2417:   /* Create localToGlobalSF as identity mapping */
2418:   {
2419:     PetscLayout map;

2421:     PetscCall(PetscLayoutCreateFromSizes(comm, nCells, PETSC_DECIDE, 1, &map));
2422:     PetscCall(PetscSFCreateFromLayouts(map, map, cellLocalToGlobalSF));
2423:     PetscCall(PetscSFSetUp(*cellLocalToGlobalSF));
2424:     PetscCall(PetscLayoutDestroy(&map));
2425:   }
2426:   PetscFunctionReturn(PETSC_SUCCESS);
2427: }

2429: static PetscErrorCode DMPlexTopologyBuildFromLayers_Private(DM dm, PetscInt depth, PlexLayer *layers, IS strataPermutation)
2430: {
2431:   const PetscInt *permArr;
2432:   PetscInt        d, nPoints;
2433:   MPI_Comm        comm;

2435:   PetscFunctionBegin;
2436:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2437:   PetscCall(ISGetIndices(strataPermutation, &permArr));

2439:   /* Count points, strata offsets and cones offsets (taking strataPermutation into account) */
2440:   {
2441:     PetscInt stratumOffset = 0;
2442:     PetscInt conesOffset   = 0;

2444:     for (d = 0; d <= depth; d++) {
2445:       const PetscInt  e = permArr[d];
2446:       const PlexLayer l = layers[e];
2447:       PetscInt        lo, n, size;

2449:       PetscCall(PetscSectionGetChart(l->coneSizesSection, &lo, &n));
2450:       PetscCall(PetscSectionGetStorageSize(l->coneSizesSection, &size));
2451:       PetscCheck(lo == 0, comm, PETSC_ERR_PLIB, "starting point should be 0 in coneSizesSection %" PetscInt_FMT, d);
2452:       l->offset      = stratumOffset;
2453:       l->conesOffset = conesOffset;
2454:       stratumOffset += n;
2455:       conesOffset += size;
2456:     }
2457:     nPoints = stratumOffset;
2458:   }

2460:   /* Set interval for all plex points */
2461:   //TODO we should store starting point of plex
2462:   PetscCall(DMPlexSetChart(dm, 0, nPoints));

2464:   /* Set up plex coneSection from layer coneSections */
2465:   {
2466:     PetscSection coneSection;

2468:     PetscCall(DMPlexGetConeSection(dm, &coneSection));
2469:     for (d = 0; d <= depth; d++) {
2470:       const PlexLayer l = layers[d];
2471:       PetscInt        n, q;

2473:       PetscCall(PetscSectionGetChart(l->coneSizesSection, NULL, &n));
2474:       for (q = 0; q < n; q++) {
2475:         const PetscInt p = l->offset + q;
2476:         PetscInt       coneSize;

2478:         PetscCall(PetscSectionGetDof(l->coneSizesSection, q, &coneSize));
2479:         PetscCall(PetscSectionSetDof(coneSection, p, coneSize));
2480:       }
2481:     }
2482:   }
2483:   //TODO this is terrible, DMSetUp_Plex() should be DMPlexSetUpSections() or so
2484:   PetscCall(DMSetUp(dm));

2486:   /* Renumber cones points from layer-global numbering to plex-local numbering */
2487:   {
2488:     PetscInt *cones, *ornts;

2490:     PetscCall(DMPlexGetCones(dm, &cones));
2491:     PetscCall(DMPlexGetConeOrientations(dm, &ornts));
2492:     for (d = 1; d <= depth; d++) {
2493:       const PlexLayer l = layers[d];
2494:       PetscInt        i, lConesSize;
2495:       PetscInt       *lCones;
2496:       const PetscInt *lOrnts;
2497:       PetscInt       *pCones = &cones[l->conesOffset];
2498:       PetscInt       *pOrnts = &ornts[l->conesOffset];

2500:       PetscCall(PetscSectionGetStorageSize(l->coneSizesSection, &lConesSize));
2501:       /* Get cones in local plex numbering */
2502:       {
2503:         ISLocalToGlobalMapping l2g;
2504:         PetscLayout            vertexLayout = l->vertexLayout;
2505:         PetscSF                vertexSF     = layers[d - 1]->l2gSF; /* vertices of this layer are cells of previous layer */
2506:         const PetscInt        *gCones;
2507:         PetscInt               lConesSize0;

2509:         PetscCall(ISGetLocalSize(l->conesIS, &lConesSize0));
2510:         PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "layer %" PetscInt_FMT " size(conesIS) = %" PetscInt_FMT " != %" PetscInt_FMT " = storageSize(coneSizesSection)", d, lConesSize0, lConesSize);
2511:         PetscCall(ISGetLocalSize(l->orientationsIS, &lConesSize0));
2512:         PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "layer %" PetscInt_FMT " size(orientationsIS) = %" PetscInt_FMT " != %" PetscInt_FMT " = storageSize(coneSizesSection)", d, lConesSize0, lConesSize);

2514:         PetscCall(PetscMalloc1(lConesSize, &lCones));
2515:         PetscCall(ISGetIndices(l->conesIS, &gCones));
2516:         PetscCall(ISLocalToGlobalMappingCreateSF(vertexSF, vertexLayout->rstart, &l2g));
2517:         PetscCall(ISGlobalToLocalMappingApply(l2g, IS_GTOLM_MASK, lConesSize, gCones, &lConesSize0, lCones));
2518:         PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "global to local does not cover all indices (%" PetscInt_FMT " of %" PetscInt_FMT ")", lConesSize0, lConesSize);
2519:         PetscCall(ISLocalToGlobalMappingDestroy(&l2g));
2520:         PetscCall(ISRestoreIndices(l->conesIS, &gCones));
2521:       }
2522:       PetscCall(ISGetIndices(l->orientationsIS, &lOrnts));
2523:       /* Set cones, need to add stratum offset */
2524:       for (i = 0; i < lConesSize; i++) {
2525:         pCones[i] = lCones[i] + layers[d - 1]->offset; /* cone points of current layer are points of previous layer */
2526:         pOrnts[i] = lOrnts[i];
2527:       }
2528:       PetscCall(PetscFree(lCones));
2529:       PetscCall(ISRestoreIndices(l->orientationsIS, &lOrnts));
2530:     }
2531:   }
2532:   PetscCall(DMPlexSymmetrize(dm));
2533:   PetscCall(DMPlexStratify(dm));
2534:   PetscCall(ISRestoreIndices(strataPermutation, &permArr));
2535:   PetscFunctionReturn(PETSC_SUCCESS);
2536: }

2538: static PetscErrorCode PlexLayerConcatenateSFs_Private(MPI_Comm comm, PetscInt depth, PlexLayer layers[], IS strataPermutation, PetscSF *overlapSF, PetscSF *l2gSF)
2539: {
2540:   PetscInt        d;
2541:   PetscSF        *osfs, *lsfs;
2542:   PetscInt       *leafOffsets;
2543:   const PetscInt *permArr;

2545:   PetscFunctionBegin;
2546:   PetscCall(ISGetIndices(strataPermutation, &permArr));
2547:   PetscCall(PetscCalloc3(depth + 1, &osfs, depth + 1, &lsfs, depth + 1, &leafOffsets));
2548:   for (d = 0; d <= depth; d++) {
2549:     const PetscInt e = permArr[d];

2551:     PetscAssert(e == layers[e]->d, PETSC_COMM_SELF, PETSC_ERR_PLIB, "assertion: e == layers[e]->d");
2552:     osfs[d]        = layers[e]->overlapSF;
2553:     lsfs[d]        = layers[e]->l2gSF;
2554:     leafOffsets[d] = layers[e]->offset;
2555:   }
2556:   PetscCall(PetscSFConcatenate(comm, depth + 1, osfs, PETSCSF_CONCATENATE_ROOTMODE_LOCAL, leafOffsets, overlapSF));
2557:   PetscCall(PetscSFConcatenate(comm, depth + 1, lsfs, PETSCSF_CONCATENATE_ROOTMODE_GLOBAL, leafOffsets, l2gSF));
2558:   PetscCall(PetscFree3(osfs, lsfs, leafOffsets));
2559:   PetscCall(ISRestoreIndices(strataPermutation, &permArr));
2560:   PetscFunctionReturn(PETSC_SUCCESS);
2561: }

2563: // Parallel load of topology
2564: static PetscErrorCode DMPlexTopologyLoad_HDF5_Private(DM dm, PetscViewer viewer, PetscSF *sfXC)
2565: {
2566:   PlexLayer  *layers;
2567:   IS          strataPermutation;
2568:   PetscLayout pointsLayout;
2569:   PetscInt    depth;
2570:   PetscInt    d;
2571:   MPI_Comm    comm;

2573:   PetscFunctionBegin;
2574:   {
2575:     PetscInt dim;

2577:     PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "depth", PETSC_INT, NULL, &depth));
2578:     PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "cell_dim", PETSC_INT, NULL, &dim));
2579:     PetscCall(DMSetDimension(dm, dim));
2580:   }
2581:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));

2583:   PetscCall(PetscInfo(dm, "Loading DM %s in parallel\n", dm->hdr.name));
2584:   {
2585:     IS spOnComm;

2587:     PetscCall(ISCreate(comm, &spOnComm));
2588:     PetscCall(PetscObjectSetName((PetscObject)spOnComm, "permutation"));
2589:     PetscCall(ISLoad(spOnComm, viewer));
2590:     /* have the same serial IS on every rank */
2591:     PetscCall(ISAllGather(spOnComm, &strataPermutation));
2592:     PetscCall(PetscObjectSetName((PetscObject)strataPermutation, ((PetscObject)spOnComm)->name));
2593:     PetscCall(ISDestroy(&spOnComm));
2594:   }

2596:   /* Create layers, load raw data for each layer */
2597:   PetscCall(PetscViewerHDF5PushGroup(viewer, "strata"));
2598:   PetscCall(PetscMalloc1(depth + 1, &layers));
2599:   for (d = depth, pointsLayout = NULL; d >= 0; pointsLayout = layers[d]->vertexLayout, d--) {
2600:     PetscCall(PlexLayerCreate_Private(&layers[d]));
2601:     PetscCall(PlexLayerLoad_Private(layers[d], viewer, d, pointsLayout));
2602:   }
2603:   PetscCall(PetscViewerHDF5PopGroup(viewer)); /* strata */

2605:   for (d = depth; d >= 0; d--) {
2606:     /* Redistribute cells and vertices for each applicable layer */
2607:     if (d < depth) PetscCall(PlexLayerDistribute_Private(layers[d], layers[d]->l2gSF));
2608:     /* Create vertex overlap SF and vertex localToGlobal SF for each applicable layer */
2609:     if (d > 0) PetscCall(PlexLayerCreateSFs_Private(layers[d], &layers[d - 1]->overlapSF, &layers[d - 1]->l2gSF));
2610:   }
2611:   /* Build trivial SFs for the cell layer as well */
2612:   PetscCall(PlexLayerCreateCellSFs_Private(layers[depth], &layers[depth]->overlapSF, &layers[depth]->l2gSF));

2614:   /* Build DMPlex topology from the layers */
2615:   PetscCall(DMPlexTopologyBuildFromLayers_Private(dm, depth, layers, strataPermutation));

2617:   /* Build overall point SF alias overlap SF */
2618:   {
2619:     PetscSF overlapSF;

2621:     PetscCall(PlexLayerConcatenateSFs_Private(comm, depth, layers, strataPermutation, &overlapSF, sfXC));
2622:     PetscCall(DMSetPointSF(dm, overlapSF));
2623:     PetscCall(PetscSFDestroy(&overlapSF));
2624:   }

2626:   for (d = depth; d >= 0; d--) PetscCall(PlexLayerDestroy(&layers[d]));
2627:   PetscCall(PetscFree(layers));
2628:   PetscCall(ISDestroy(&strataPermutation));
2629:   PetscFunctionReturn(PETSC_SUCCESS);
2630: }

2632: PetscErrorCode DMPlexTopologyLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF *sfXC)
2633: {
2634:   DMPlexStorageVersion version;
2635:   const char          *topologydm_name;
2636:   char                 group[PETSC_MAX_PATH_LEN];
2637:   PetscSF              sfwork = NULL;

2639:   PetscFunctionBegin;
2640:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2641:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2642:   if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
2643:     PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s", topologydm_name));
2644:   } else {
2645:     PetscCall(PetscStrncpy(group, "/", sizeof(group)));
2646:   }
2647:   PetscCall(PetscViewerHDF5PushGroup(viewer, group));

2649:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topology"));
2650:   if (version->major < 3) {
2651:     PetscCall(DMPlexTopologyLoad_HDF5_Legacy_Private(dm, viewer, &sfwork));
2652:   } else {
2653:     /* since DMPlexStorageVersion 3.0.0 */
2654:     PetscCall(DMPlexTopologyLoad_HDF5_Private(dm, viewer, &sfwork));
2655:   }
2656:   PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "topology" */

2658:   if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
2659:     DM          distdm;
2660:     PetscSF     distsf;
2661:     const char *distribution_name;
2662:     PetscBool   exists;

2664:     PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
2665:     /* this group is guaranteed to be present since DMPlexStorageVersion 2.1.0 */
2666:     PetscCall(PetscViewerHDF5PushGroup(viewer, "distributions"));
2667:     PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &exists));
2668:     if (exists) {
2669:       PetscCall(PetscViewerHDF5PushGroup(viewer, distribution_name));
2670:       PetscCall(DMPlexDistributionLoad_HDF5_Private(dm, viewer, sfwork, &distsf, &distdm));
2671:       if (distdm) {
2672:         PetscCall(DMPlexReplace_Internal(dm, &distdm));
2673:         PetscCall(PetscSFDestroy(&sfwork));
2674:         sfwork = distsf;
2675:       }
2676:       PetscCall(PetscViewerHDF5PopGroup(viewer)); /* distribution_name */
2677:     }
2678:     PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "distributions" */
2679:   }
2680:   if (sfXC) {
2681:     *sfXC = sfwork;
2682:   } else {
2683:     PetscCall(PetscSFDestroy(&sfwork));
2684:   }

2686:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2687:   PetscFunctionReturn(PETSC_SUCCESS);
2688: }

2690: /* If the file is old, it not only has different path to the coordinates, but   */
2691: /* does not contain coordinateDMs, so must fall back to the old implementation. */
2692: static PetscErrorCode DMPlexCoordinatesLoad_HDF5_Legacy_Private(DM dm, PetscViewer viewer)
2693: {
2694:   PetscSection coordSection;
2695:   Vec          coordinates;
2696:   PetscReal    lengthScale;
2697:   PetscInt     spatialDim, N, numVertices, vStart, vEnd, v;
2698:   PetscMPIInt  rank;

2700:   PetscFunctionBegin;
2701:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2702:   /* Read geometry */
2703:   PetscCall(PetscViewerHDF5PushGroup(viewer, "/geometry"));
2704:   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
2705:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "vertices"));
2706:   {
2707:     /* Force serial load */
2708:     PetscCall(PetscViewerHDF5ReadSizes(viewer, "vertices", &spatialDim, &N));
2709:     PetscCall(VecSetSizes(coordinates, rank == 0 ? N : 0, N));
2710:     PetscCall(VecSetBlockSize(coordinates, spatialDim));
2711:   }
2712:   PetscCall(VecLoad(coordinates, viewer));
2713:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2714:   PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
2715:   PetscCall(VecScale(coordinates, 1.0 / lengthScale));
2716:   PetscCall(VecGetLocalSize(coordinates, &numVertices));
2717:   PetscCall(VecGetBlockSize(coordinates, &spatialDim));
2718:   numVertices /= spatialDim;
2719:   /* Create coordinates */
2720:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2721:   PetscCheck(numVertices == vEnd - vStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coordinates loaded %" PetscInt_FMT " does not match number of vertices %" PetscInt_FMT, numVertices, vEnd - vStart);
2722:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2723:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2724:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spatialDim));
2725:   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
2726:   for (v = vStart; v < vEnd; ++v) {
2727:     PetscCall(PetscSectionSetDof(coordSection, v, spatialDim));
2728:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spatialDim));
2729:   }
2730:   PetscCall(PetscSectionSetUp(coordSection));
2731:   PetscCall(DMSetCoordinates(dm, coordinates));
2732:   PetscCall(VecDestroy(&coordinates));
2733:   PetscFunctionReturn(PETSC_SUCCESS);
2734: }

2736: PetscErrorCode DMPlexCoordinatesLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF sfXC)
2737: {
2738:   DMPlexStorageVersion version;
2739:   DM                   cdm;
2740:   Vec                  coords;
2741:   PetscInt             blockSize;
2742:   PetscReal            lengthScale;
2743:   PetscSF              lsf;
2744:   const char          *topologydm_name;
2745:   char                *coordinatedm_name, *coordinates_name;

2747:   PetscFunctionBegin;
2748:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2749:   if (!DMPlexStorageVersionGE(version, 2, 0, 0)) {
2750:     PetscCall(DMPlexCoordinatesLoad_HDF5_Legacy_Private(dm, viewer));
2751:     PetscFunctionReturn(PETSC_SUCCESS);
2752:   }
2753:   /* else: since DMPlexStorageVersion 2.0.0 */
2754:   PetscCheck(sfXC, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_NULL, "PetscSF must be given for parallel load");
2755:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2756:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
2757:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
2758:   PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "coordinateDMName", PETSC_STRING, NULL, &coordinatedm_name));
2759:   PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "coordinatesName", PETSC_STRING, NULL, &coordinates_name));
2760:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2761:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2762:   PetscCall(DMGetCoordinateDM(dm, &cdm));
2763:   PetscCall(PetscObjectSetName((PetscObject)cdm, coordinatedm_name));
2764:   PetscCall(PetscFree(coordinatedm_name));
2765:   /* lsf: on-disk data -> in-memory local vector associated with cdm's local section */
2766:   PetscCall(DMPlexSectionLoad(dm, viewer, cdm, sfXC, NULL, &lsf));
2767:   PetscCall(DMCreateLocalVector(cdm, &coords));
2768:   PetscCall(PetscObjectSetName((PetscObject)coords, coordinates_name));
2769:   PetscCall(PetscFree(coordinates_name));
2770:   PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
2771:   PetscCall(DMPlexLocalVectorLoad(dm, viewer, cdm, lsf, coords));
2772:   PetscCall(PetscViewerPopFormat(viewer));
2773:   PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
2774:   PetscCall(VecScale(coords, 1.0 / lengthScale));
2775:   PetscCall(DMSetCoordinatesLocal(dm, coords));
2776:   PetscCall(VecGetBlockSize(coords, &blockSize));
2777:   PetscCall(DMSetCoordinateDim(dm, blockSize));
2778:   PetscCall(VecDestroy(&coords));
2779:   PetscCall(PetscSFDestroy(&lsf));
2780:   PetscFunctionReturn(PETSC_SUCCESS);
2781: }

2783: PetscErrorCode DMPlexLoad_HDF5_Internal(DM dm, PetscViewer viewer)
2784: {
2785:   DMPlexStorageVersion version;

2787:   PetscFunctionBegin;
2788:   PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2789:   PetscCall(PetscInfo(dm, "Loading DM %s storage version %d.%d.%d\n", dm->hdr.name, version->major, version->minor, version->subminor));
2790:   if (!DMPlexStorageVersionGE(version, 2, 0, 0)) {
2791:     PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, NULL));
2792:     PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, NULL));
2793:     PetscCall(DMPlexCoordinatesLoad_HDF5_Legacy_Private(dm, viewer));
2794:   } else {
2795:     PetscSF sfXC;

2797:     /* since DMPlexStorageVersion 2.0.0 */
2798:     PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, &sfXC));
2799:     PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, sfXC));
2800:     PetscCall(DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, sfXC));
2801:     PetscCall(PetscSFDestroy(&sfXC));
2802:   }
2803:   PetscFunctionReturn(PETSC_SUCCESS);
2804: }

2806: static PetscErrorCode DMPlexSectionLoad_HDF5_Internal_CreateDataSF(PetscSection rootSection, PetscLayout layout, PetscInt globalOffsets[], PetscSection leafSection, PetscSF *sectionSF)
2807: {
2808:   MPI_Comm  comm;
2809:   PetscInt  pStart, pEnd, p, m;
2810:   PetscInt *goffs, *ilocal;
2811:   PetscBool rootIncludeConstraints, leafIncludeConstraints;

2813:   PetscFunctionBegin;
2814:   PetscCall(PetscObjectGetComm((PetscObject)leafSection, &comm));
2815:   PetscCall(PetscSectionGetChart(leafSection, &pStart, &pEnd));
2816:   PetscCall(PetscSectionGetIncludesConstraints(rootSection, &rootIncludeConstraints));
2817:   PetscCall(PetscSectionGetIncludesConstraints(leafSection, &leafIncludeConstraints));
2818:   if (rootIncludeConstraints && leafIncludeConstraints) PetscCall(PetscSectionGetStorageSize(leafSection, &m));
2819:   else PetscCall(PetscSectionGetConstrainedStorageSize(leafSection, &m));
2820:   PetscCall(PetscMalloc1(m, &ilocal));
2821:   PetscCall(PetscMalloc1(m, &goffs));
2822:   /* Currently, PetscSFDistributeSection() returns globalOffsets[] only */
2823:   /* for the top-level section (not for each field), so one must have   */
2824:   /* rootSection->pointMajor == PETSC_TRUE.                             */
2825:   PetscCheck(rootSection->pointMajor, PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for field major ordering");
2826:   /* Currently, we also assume that leafSection->pointMajor == PETSC_TRUE. */
2827:   PetscCheck(leafSection->pointMajor, PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for field major ordering");
2828:   for (p = pStart, m = 0; p < pEnd; ++p) {
2829:     PetscInt        dof, cdof, i, j, off, goff;
2830:     const PetscInt *cinds;

2832:     PetscCall(PetscSectionGetDof(leafSection, p, &dof));
2833:     if (dof < 0) continue;
2834:     goff = globalOffsets[p - pStart];
2835:     PetscCall(PetscSectionGetOffset(leafSection, p, &off));
2836:     PetscCall(PetscSectionGetConstraintDof(leafSection, p, &cdof));
2837:     PetscCall(PetscSectionGetConstraintIndices(leafSection, p, &cinds));
2838:     for (i = 0, j = 0; i < dof; ++i) {
2839:       PetscBool constrained = (PetscBool)(j < cdof && i == cinds[j]);

2841:       if (!constrained || (leafIncludeConstraints && rootIncludeConstraints)) {
2842:         ilocal[m]  = off++;
2843:         goffs[m++] = goff++;
2844:       } else if (leafIncludeConstraints && !rootIncludeConstraints) ++off;
2845:       else if (!leafIncludeConstraints && rootIncludeConstraints) ++goff;
2846:       if (constrained) ++j;
2847:     }
2848:   }
2849:   PetscCall(PetscSFCreate(comm, sectionSF));
2850:   PetscCall(PetscSFSetFromOptions(*sectionSF));
2851:   PetscCall(PetscSFSetGraphLayout(*sectionSF, layout, m, ilocal, PETSC_OWN_POINTER, goffs));
2852:   PetscCall(PetscFree(goffs));
2853:   PetscFunctionReturn(PETSC_SUCCESS);
2854: }

2856: PetscErrorCode DMPlexSectionLoad_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sfXB, PetscSF *gsf, PetscSF *lsf)
2857: {
2858:   MPI_Comm     comm;
2859:   PetscMPIInt  size, rank;
2860:   const char  *topologydm_name;
2861:   const char  *sectiondm_name;
2862:   PetscSection sectionA, sectionB;
2863:   PetscBool    has;
2864:   PetscInt     nX, n, i;
2865:   PetscSF      sfAB;

2867:   PetscFunctionBegin;
2868:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2869:   PetscCallMPI(MPI_Comm_size(comm, &size));
2870:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
2871:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2872:   PetscCall(PetscObjectGetName((PetscObject)sectiondm, &sectiondm_name));
2873:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
2874:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
2875:   PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
2876:   PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
2877:   /* A: on-disk points                        */
2878:   /* X: list of global point numbers, [0, NX) */
2879:   /* B: plex points                           */
2880:   /* Load raw section (sectionA)              */
2881:   PetscCall(PetscSectionCreate(comm, &sectionA));
2882:   PetscCall(PetscViewerHDF5HasGroup(viewer, "section", &has));
2883:   if (has) PetscCall(PetscSectionLoad(sectionA, viewer));
2884:   else {
2885:     // TODO If section is missing, create the default affine section with dim dofs on each vertex. Use PetscSplitOwnership() to split vertices.
2886:     //   How do I know the total number of vertices?
2887:     PetscInt dim, Nf = 1, Nv, nv = PETSC_DECIDE;

2889:     PetscCall(DMGetDimension(dm, &dim));
2890:     PetscCall(DMPlexGetDepthStratumGlobalSize(dm, 0, &Nv));
2891:     PetscCall(PetscSectionSetNumFields(sectionA, Nf));
2892:     PetscCall(PetscSectionSetFieldName(sectionA, 0, "Cartesian"));
2893:     PetscCall(PetscSectionSetFieldComponents(sectionA, 0, dim));
2894:     for (PetscInt c = 0; c < dim; ++c) {
2895:       char axis = 'X' + (char)c;

2897:       PetscCall(PetscSectionSetComponentName(sectionA, 0, c, &axis));
2898:     }
2899:     PetscCall(PetscSplitOwnership(comm, &nv, &Nv));
2900:     PetscCall(PetscSectionSetChart(sectionA, 0, nv));
2901:     for (PetscInt p = 0; p < nv; ++p) {
2902:       PetscCall(PetscSectionSetDof(sectionA, p, dim));
2903:       PetscCall(PetscSectionSetFieldDof(sectionA, p, 0, dim));
2904:     }
2905:     PetscCall(PetscSectionSetUp(sectionA));
2906:   }
2907:   PetscCall(PetscSectionGetChart(sectionA, NULL, &n));
2908:   /* Create sfAB: A -> B */
2909:   #if defined(PETSC_USE_DEBUG)
2910:   {
2911:     PetscInt N, N1;

2913:     PetscCall(PetscViewerHDF5ReadSizes(viewer, "order", NULL, &N1));
2914:     PetscCallMPI(MPIU_Allreduce(&n, &N, 1, MPIU_INT, MPI_SUM, comm));
2915:     PetscCheck(N1 == N, comm, PETSC_ERR_ARG_SIZ, "Mismatching sizes: on-disk order array size (%" PetscInt_FMT ") != number of loaded section points (%" PetscInt_FMT ")", N1, N);
2916:   }
2917:   #endif
2918:   {
2919:     IS              orderIS;
2920:     const PetscInt *gpoints;
2921:     PetscSF         sfXA, sfAX;
2922:     PetscLayout     layout;
2923:     PetscSFNode    *owners, *buffer;
2924:     PetscInt        nleaves;
2925:     PetscInt       *ilocal;
2926:     PetscSFNode    *iremote;

2928:     /* Create sfAX: A -> X */
2929:     PetscCall(ISCreate(comm, &orderIS));
2930:     PetscCall(PetscObjectSetName((PetscObject)orderIS, "order"));
2931:     PetscCall(PetscLayoutSetLocalSize(orderIS->map, n));
2932:     PetscCall(ISLoad(orderIS, viewer));
2933:     PetscCall(PetscLayoutCreate(comm, &layout));
2934:     PetscCall(PetscSFGetGraph(sfXB, &nX, NULL, NULL, NULL));
2935:     PetscCall(PetscLayoutSetLocalSize(layout, nX));
2936:     PetscCall(PetscLayoutSetBlockSize(layout, 1));
2937:     PetscCall(PetscLayoutSetUp(layout));
2938:     PetscCall(PetscSFCreate(comm, &sfXA));
2939:     PetscCall(ISGetIndices(orderIS, &gpoints));
2940:     PetscCall(PetscSFSetGraphLayout(sfXA, layout, n, NULL, PETSC_OWN_POINTER, gpoints));
2941:     PetscCall(ISRestoreIndices(orderIS, &gpoints));
2942:     PetscCall(ISDestroy(&orderIS));
2943:     PetscCall(PetscLayoutDestroy(&layout));
2944:     PetscCall(PetscMalloc1(n, &owners));
2945:     PetscCall(PetscMalloc1(nX, &buffer));
2946:     for (i = 0; i < n; ++i) {
2947:       owners[i].rank  = rank;
2948:       owners[i].index = i;
2949:     }
2950:     for (i = 0; i < nX; ++i) {
2951:       buffer[i].rank  = -1;
2952:       buffer[i].index = -1;
2953:     }
2954:     PetscCall(PetscSFReduceBegin(sfXA, MPIU_SF_NODE, owners, buffer, MPI_MAXLOC));
2955:     PetscCall(PetscSFReduceEnd(sfXA, MPIU_SF_NODE, owners, buffer, MPI_MAXLOC));
2956:     PetscCall(PetscSFDestroy(&sfXA));
2957:     PetscCall(PetscFree(owners));
2958:     for (i = 0, nleaves = 0; i < nX; ++i)
2959:       if (buffer[i].rank >= 0) nleaves++;
2960:     PetscCall(PetscMalloc1(nleaves, &ilocal));
2961:     PetscCall(PetscMalloc1(nleaves, &iremote));
2962:     for (i = 0, nleaves = 0; i < nX; ++i) {
2963:       if (buffer[i].rank >= 0) {
2964:         ilocal[nleaves]        = i;
2965:         iremote[nleaves].rank  = buffer[i].rank;
2966:         iremote[nleaves].index = buffer[i].index;
2967:         nleaves++;
2968:       }
2969:     }
2970:     PetscCall(PetscFree(buffer));
2971:     PetscCall(PetscSFCreate(comm, &sfAX));
2972:     PetscCall(PetscSFSetFromOptions(sfAX));
2973:     PetscCall(PetscSFSetGraph(sfAX, n, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
2974:     PetscCall(PetscSFCompose(sfAX, sfXB, &sfAB));
2975:     PetscCall(PetscSFDestroy(&sfAX));
2976:   }
2977:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2978:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2979:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2980:   PetscCall(PetscViewerHDF5PopGroup(viewer));
2981:   /* Create plex section (sectionB) */
2982:   PetscCall(DMGetLocalSection(sectiondm, &sectionB));
2983:   if (lsf || gsf) {
2984:     PetscLayout layout;
2985:     PetscInt    M, m;
2986:     PetscInt   *offsetsA;
2987:     PetscBool   includesConstraintsA;

2989:     PetscCall(PetscSFDistributeSection(sfAB, sectionA, &offsetsA, sectionB));
2990:     PetscCall(PetscSectionGetIncludesConstraints(sectionA, &includesConstraintsA));
2991:     if (includesConstraintsA) PetscCall(PetscSectionGetStorageSize(sectionA, &m));
2992:     else PetscCall(PetscSectionGetConstrainedStorageSize(sectionA, &m));
2993:     PetscCallMPI(MPIU_Allreduce(&m, &M, 1, MPIU_INT, MPI_SUM, comm));
2994:     PetscCall(PetscLayoutCreate(comm, &layout));
2995:     PetscCall(PetscLayoutSetSize(layout, M));
2996:     PetscCall(PetscLayoutSetUp(layout));
2997:     if (lsf) {
2998:       PetscSF lsfABdata;

3000:       PetscCall(DMPlexSectionLoad_HDF5_Internal_CreateDataSF(sectionA, layout, offsetsA, sectionB, &lsfABdata));
3001:       *lsf = lsfABdata;
3002:     }
3003:     if (gsf) {
3004:       PetscSection gsectionB, gsectionB1;
3005:       PetscBool    includesConstraintsB;
3006:       PetscSF      gsfABdata, pointsf;

3008:       PetscCall(DMGetGlobalSection(sectiondm, &gsectionB1));
3009:       PetscCall(PetscSectionGetIncludesConstraints(gsectionB1, &includesConstraintsB));
3010:       PetscCall(DMGetPointSF(sectiondm, &pointsf));
3011:       PetscCall(PetscSectionCreateGlobalSection(sectionB, pointsf, PETSC_TRUE, includesConstraintsB, PETSC_TRUE, &gsectionB));
3012:       PetscCall(DMPlexSectionLoad_HDF5_Internal_CreateDataSF(sectionA, layout, offsetsA, gsectionB, &gsfABdata));
3013:       PetscCall(PetscSectionDestroy(&gsectionB));
3014:       *gsf = gsfABdata;
3015:     }
3016:     PetscCall(PetscLayoutDestroy(&layout));
3017:     PetscCall(PetscFree(offsetsA));
3018:   } else {
3019:     PetscCall(PetscSFDistributeSection(sfAB, sectionA, NULL, sectionB));
3020:   }
3021:   PetscCall(PetscSFDestroy(&sfAB));
3022:   PetscCall(PetscSectionDestroy(&sectionA));
3023:   PetscFunctionReturn(PETSC_SUCCESS);
3024: }

3026: PetscErrorCode DMPlexVecLoad_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
3027: {
3028:   MPI_Comm           comm;
3029:   const char        *topologydm_name;
3030:   const char        *sectiondm_name;
3031:   const char        *vec_name;
3032:   Vec                vecA;
3033:   PetscInt           mA, m, bs;
3034:   const PetscInt    *ilocal;
3035:   const PetscScalar *src;
3036:   PetscScalar       *dest;

3038:   PetscFunctionBegin;
3039:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3040:   PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
3041:   PetscCall(PetscObjectGetName((PetscObject)sectiondm, &sectiondm_name));
3042:   PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
3043:   PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
3044:   PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
3045:   PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
3046:   PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
3047:   PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
3048:   PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
3049:   PetscCall(VecCreate(comm, &vecA));
3050:   PetscCall(PetscObjectSetName((PetscObject)vecA, vec_name));
3051:   PetscCall(PetscSFGetGraph(sf, &mA, &m, &ilocal, NULL));
3052:   /* Check consistency */
3053:   {
3054:     PetscSF  pointsf, pointsf1;
3055:     PetscInt m1, i, j;

3057:     PetscCall(DMGetPointSF(dm, &pointsf));
3058:     PetscCall(DMGetPointSF(sectiondm, &pointsf1));
3059:     PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
3060:   #if defined(PETSC_USE_DEBUG)
3061:     {
3062:       PetscInt MA, MA1;

3064:       PetscCallMPI(MPIU_Allreduce(&mA, &MA, 1, MPIU_INT, MPI_SUM, comm));
3065:       PetscCall(PetscViewerHDF5ReadSizes(viewer, vec_name, NULL, &MA1));
3066:       PetscCheck(MA1 == MA, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Total SF root size (%" PetscInt_FMT ") != On-disk vector data size (%" PetscInt_FMT ")", MA, MA1);
3067:     }
3068:   #endif
3069:     PetscCall(VecGetLocalSize(vec, &m1));
3070:     PetscCheck(m1 >= m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Target vector size (%" PetscInt_FMT ") < SF leaf size (%" PetscInt_FMT ")", m1, m);
3071:     for (i = 0; i < m; ++i) {
3072:       j = ilocal ? ilocal[i] : i;
3073:       PetscCheck(j >= 0 && j < m1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Leaf's %" PetscInt_FMT "-th index, %" PetscInt_FMT ", not in [%" PetscInt_FMT ", %" PetscInt_FMT ")", i, j, (PetscInt)0, m1);
3074:     }
3075:   }
3076:   PetscCall(VecSetSizes(vecA, mA, PETSC_DECIDE));
3077:   PetscCall(VecLoad(vecA, viewer));
3078:   PetscCall(VecGetArrayRead(vecA, &src));
3079:   PetscCall(VecGetArray(vec, &dest));
3080:   PetscCall(PetscSFBcastBegin(sf, MPIU_SCALAR, src, dest, MPI_REPLACE));
3081:   PetscCall(PetscSFBcastEnd(sf, MPIU_SCALAR, src, dest, MPI_REPLACE));
3082:   PetscCall(VecRestoreArray(vec, &dest));
3083:   PetscCall(VecRestoreArrayRead(vecA, &src));
3084:   PetscCall(VecDestroy(&vecA));
3085:   PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "blockSize", PETSC_INT, NULL, (void *)&bs));
3086:   PetscCall(VecSetBlockSize(vec, bs));
3087:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3088:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3089:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3090:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3091:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3092:   PetscCall(PetscViewerHDF5PopGroup(viewer));
3093:   PetscFunctionReturn(PETSC_SUCCESS);
3094: }
3095: #endif