Actual source code: plexhdf5.c
1: #include <petsc/private/dmpleximpl.h>
2: #include <petsc/private/isimpl.h>
3: #include <petsc/private/vecimpl.h>
4: #include <petsc/private/viewerhdf5impl.h>
5: #include <petsclayouthdf5.h>
7: /* Logging support */
8: PetscLogEvent DMPLEX_DistributionView, DMPLEX_DistributionLoad;
10: #if defined(PETSC_HAVE_HDF5)
11: static PetscErrorCode PetscViewerParseVersion_Private(PetscViewer, const char[], DMPlexStorageVersion *);
12: static PetscErrorCode PetscViewerCheckVersion_Private(PetscViewer, DMPlexStorageVersion);
13: static PetscErrorCode PetscViewerAttachVersion_Private(PetscViewer, const char[], DMPlexStorageVersion);
14: static PetscErrorCode PetscViewerGetAttachedVersion_Private(PetscViewer, const char[], DMPlexStorageVersion *);
16: PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
18: static PetscErrorCode PetscViewerPrintVersion_Private(PetscViewer viewer, DMPlexStorageVersion version, char str[], size_t len)
19: {
20: PetscFunctionBegin;
21: PetscCall(PetscViewerCheckVersion_Private(viewer, version));
22: PetscCall(PetscSNPrintf(str, len, "%d.%d.%d", version->major, version->minor, version->subminor));
23: PetscFunctionReturn(PETSC_SUCCESS);
24: }
26: static PetscErrorCode PetscViewerParseVersion_Private(PetscViewer viewer, const char str[], DMPlexStorageVersion *version)
27: {
28: PetscToken t;
29: char *ts;
30: PetscInt i;
31: PetscInt ti[3];
32: DMPlexStorageVersion v;
34: PetscFunctionBegin;
35: PetscCall(PetscTokenCreate(str, '.', &t));
36: for (i = 0; i < 3; i++) {
37: PetscCall(PetscTokenFind(t, &ts));
38: PetscCheck(ts, PetscObjectComm((PetscObject)viewer), PETSC_ERR_ARG_WRONG, "Malformed version string %s", str);
39: PetscCall(PetscOptionsStringToInt(ts, &ti[i]));
40: }
41: PetscCall(PetscTokenFind(t, &ts));
42: PetscCheck(!ts, PetscObjectComm((PetscObject)viewer), PETSC_ERR_ARG_WRONG, "Malformed version string %s", str);
43: PetscCall(PetscTokenDestroy(&t));
44: PetscCall(PetscNew(&v));
45: v->major = (int)ti[0];
46: v->minor = (int)ti[1];
47: v->subminor = (int)ti[2];
48: PetscCall(PetscViewerCheckVersion_Private(viewer, v));
49: *version = v;
50: PetscFunctionReturn(PETSC_SUCCESS);
51: }
53: static PetscErrorCode PetscViewerAttachVersion_Private(PetscViewer viewer, const char key[], DMPlexStorageVersion v)
54: {
55: PetscFunctionBegin;
56: PetscCall(PetscObjectContainerCompose((PetscObject)viewer, key, v, PetscContainerUserDestroyDefault));
57: PetscFunctionReturn(PETSC_SUCCESS);
58: }
60: static PetscErrorCode PetscViewerGetAttachedVersion_Private(PetscViewer viewer, const char key[], DMPlexStorageVersion *v)
61: {
62: PetscContainer cont;
64: PetscFunctionBegin;
65: PetscCall(PetscObjectQuery((PetscObject)viewer, key, (PetscObject *)&cont));
66: *v = NULL;
67: if (cont) PetscCall(PetscContainerGetPointer(cont, (void **)v));
68: PetscFunctionReturn(PETSC_SUCCESS);
69: }
71: /*
72: Version log:
73: 1.0.0 legacy version (default if no "dmplex_storage_version" attribute found in file)
74: 1.1.0 legacy version, but output VIZ by default
75: 2.0.0 introduce versioning and multiple topologies
76: 2.1.0 introduce distributions
77: 3.0.0 new checkpointing format in Firedrake paper
78: 3.1.0 new format with IS compression
79: */
80: static PetscErrorCode PetscViewerCheckVersion_Private(PetscViewer viewer, DMPlexStorageVersion version)
81: {
82: PetscBool valid = PETSC_FALSE;
84: PetscFunctionBegin;
85: switch (version->major) {
86: case 1:
87: switch (version->minor) {
88: case 0:
89: switch (version->subminor) {
90: case 0:
91: valid = PETSC_TRUE;
92: break;
93: }
94: break;
95: case 1:
96: switch (version->subminor) {
97: case 0:
98: valid = PETSC_TRUE;
99: break;
100: }
101: break;
102: }
103: break;
104: case 2:
105: switch (version->minor) {
106: case 0:
107: switch (version->subminor) {
108: case 0:
109: valid = PETSC_TRUE;
110: break;
111: }
112: break;
113: case 1:
114: switch (version->subminor) {
115: case 0:
116: valid = PETSC_TRUE;
117: break;
118: }
119: break;
120: }
121: break;
122: case 3:
123: switch (version->minor) {
124: case 0:
125: switch (version->subminor) {
126: case 0:
127: valid = PETSC_TRUE;
128: break;
129: }
130: break;
131: case 1:
132: switch (version->subminor) {
133: case 0:
134: valid = PETSC_TRUE;
135: break;
136: }
137: break;
138: }
139: break;
140: }
141: PetscCheck(valid, PetscObjectComm((PetscObject)viewer), PETSC_ERR_SUP, "DMPlexStorageVersion %d.%d.%d not supported", version->major, version->minor, version->subminor);
142: PetscFunctionReturn(PETSC_SUCCESS);
143: }
145: static inline PetscBool DMPlexStorageVersionEQ(DMPlexStorageVersion version, int major, int minor, int subminor)
146: {
147: return (PetscBool)(version->major == major && version->minor == minor && version->subminor == subminor);
148: }
150: static inline PetscBool DMPlexStorageVersionGE(DMPlexStorageVersion version, int major, int minor, int subminor)
151: {
152: return (PetscBool)((version->major == major && version->minor == minor && version->subminor >= subminor) || (version->major == major && version->minor > minor) || (version->major > major));
153: }
155: /*@C
156: PetscViewerHDF5SetDMPlexStorageVersionWriting - Set the storage version for writing
158: Logically collective
160: Input Parameters:
161: + viewer - The `PetscViewer`
162: - version - The storage format version
164: Level: advanced
166: Note:
167: The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.
169: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`
170: @*/
171: PetscErrorCode PetscViewerHDF5SetDMPlexStorageVersionWriting(PetscViewer viewer, DMPlexStorageVersion version)
172: {
173: const char ATTR_NAME[] = "dmplex_storage_version";
174: DMPlexStorageVersion viewerVersion;
175: PetscBool fileHasVersion;
176: char fileVersion[16], versionStr[16], viewerVersionStr[16];
178: PetscFunctionBegin;
180: PetscAssertPointer(version, 2);
181: PetscCall(PetscViewerPrintVersion_Private(viewer, version, versionStr, 16));
182: PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, &viewerVersion));
183: if (viewerVersion) {
184: PetscBool flg;
186: PetscCall(PetscViewerPrintVersion_Private(viewer, viewerVersion, viewerVersionStr, 16));
187: PetscCall(PetscStrcmp(versionStr, viewerVersionStr, &flg));
188: PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but viewer already has version %s - cannot mix versions", versionStr, viewerVersionStr);
189: }
191: PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
192: if (fileHasVersion) {
193: PetscBool flg;
194: char *tmp;
196: PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &tmp));
197: PetscCall(PetscStrncpy(fileVersion, tmp, sizeof(fileVersion)));
198: PetscCall(PetscFree(tmp));
199: PetscCall(PetscStrcmp(fileVersion, versionStr, &flg));
200: PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", versionStr, fileVersion);
201: } else {
202: PetscCall(PetscViewerHDF5WriteAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, versionStr));
203: }
204: PetscCall(PetscNew(&viewerVersion));
205: viewerVersion->major = version->major;
206: viewerVersion->minor = version->minor;
207: viewerVersion->subminor = version->subminor;
208: PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, viewerVersion));
209: PetscFunctionReturn(PETSC_SUCCESS);
210: }
212: /*@C
213: PetscViewerHDF5GetDMPlexStorageVersionWriting - Get the storage version for writing
215: Logically collective
217: Input Parameter:
218: . viewer - The `PetscViewer`
220: Output Parameter:
221: . version - The storage format version
223: Options Database Keys:
224: . -dm_plex_view_hdf5_storage_version <num> - Overrides the storage format version
226: Level: advanced
228: Note:
229: The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.
231: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`
232: @*/
233: PetscErrorCode PetscViewerHDF5GetDMPlexStorageVersionWriting(PetscViewer viewer, DMPlexStorageVersion *version)
234: {
235: const char ATTR_NAME[] = "dmplex_storage_version";
236: PetscBool fileHasVersion;
237: char optVersion[16], fileVersion[16];
239: PetscFunctionBegin;
241: PetscAssertPointer(version, 2);
242: PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, version));
243: if (*version) PetscFunctionReturn(PETSC_SUCCESS);
245: PetscCall(PetscStrncpy(fileVersion, DMPLEX_STORAGE_VERSION_STABLE, sizeof(fileVersion)));
246: PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
247: if (fileHasVersion) {
248: char *tmp;
250: PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &tmp));
251: PetscCall(PetscStrncpy(fileVersion, tmp, sizeof(fileVersion)));
252: PetscCall(PetscFree(tmp));
253: }
254: PetscCall(PetscStrncpy(optVersion, fileVersion, sizeof(optVersion)));
255: PetscObjectOptionsBegin((PetscObject)viewer);
256: PetscCall(PetscOptionsString("-dm_plex_view_hdf5_storage_version", "DMPlex HDF5 viewer storage version", NULL, optVersion, optVersion, sizeof(optVersion), NULL));
257: PetscOptionsEnd();
258: if (!fileHasVersion) {
259: PetscCall(PetscViewerHDF5WriteAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, optVersion));
260: } else {
261: PetscBool flg;
263: PetscCall(PetscStrcmp(fileVersion, optVersion, &flg));
264: PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", optVersion, fileVersion);
265: }
266: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "petsc_version_git", PETSC_STRING, PETSC_VERSION_GIT));
267: PetscCall(PetscViewerParseVersion_Private(viewer, optVersion, version));
268: PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_WRITING_KEY, *version));
269: PetscFunctionReturn(PETSC_SUCCESS);
270: }
272: /*@C
273: PetscViewerHDF5SetDMPlexStorageVersionReading - Set the storage version for reading
275: Logically collective
277: Input Parameters:
278: + viewer - The `PetscViewer`
279: - version - The storage format version
281: Level: advanced
283: Note:
284: The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.
286: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5GetDMPlexStorageVersionReading()`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`
287: @*/
288: PetscErrorCode PetscViewerHDF5SetDMPlexStorageVersionReading(PetscViewer viewer, DMPlexStorageVersion version)
289: {
290: const char ATTR_NAME[] = "dmplex_storage_version";
291: DMPlexStorageVersion viewerVersion;
292: PetscBool fileHasVersion;
293: char versionStr[16], viewerVersionStr[16];
295: PetscFunctionBegin;
297: PetscAssertPointer(version, 2);
298: PetscCall(PetscViewerPrintVersion_Private(viewer, version, versionStr, 16));
299: PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, &viewerVersion));
300: if (viewerVersion) {
301: PetscBool flg;
303: PetscCall(PetscViewerPrintVersion_Private(viewer, viewerVersion, viewerVersionStr, 16));
304: PetscCall(PetscStrcmp(versionStr, viewerVersionStr, &flg));
305: PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but viewer already has version %s - cannot mix versions", versionStr, viewerVersionStr);
306: }
308: PetscCall(PetscViewerHDF5HasAttribute(viewer, NULL, ATTR_NAME, &fileHasVersion));
309: if (fileHasVersion) {
310: char *fileVersion;
311: PetscBool flg;
313: PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, NULL, &fileVersion));
314: PetscCall(PetscStrcmp(fileVersion, versionStr, &flg));
315: PetscCheck(flg, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "User requested DMPlex storage version %s but file already has version %s - cannot mix versions", versionStr, fileVersion);
316: PetscCall(PetscFree(fileVersion));
317: }
318: PetscCall(PetscNew(&viewerVersion));
319: viewerVersion->major = version->major;
320: viewerVersion->minor = version->minor;
321: viewerVersion->subminor = version->subminor;
322: PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, viewerVersion));
323: PetscFunctionReturn(PETSC_SUCCESS);
324: }
326: /*@C
327: PetscViewerHDF5GetDMPlexStorageVersionReading - Get the storage version for reading
329: Logically collective
331: Input Parameter:
332: . viewer - The `PetscViewer`
334: Output Parameter:
335: . version - The storage format version
337: Options Database Keys:
338: . -dm_plex_view_hdf5_storage_version <num> - Overrides the storage format version
340: Level: advanced
342: Note:
343: The version has major, minor, and subminor integers. Parallel operations are only available for version 3.0.0.
345: .seealso: [](ch_dmbase), `DM`, `PetscViewerHDF5SetDMPlexStorageVersionReading()`, `PetscViewerHDF5GetDMPlexStorageVersionWriting()`, `PetscViewerHDF5SetDMPlexStorageVersionWriting()`
346: @*/
347: PetscErrorCode PetscViewerHDF5GetDMPlexStorageVersionReading(PetscViewer viewer, DMPlexStorageVersion *version)
348: {
349: const char ATTR_NAME[] = "dmplex_storage_version";
350: char *defaultVersion;
351: char *versionString;
353: PetscFunctionBegin;
355: PetscAssertPointer(version, 2);
356: PetscCall(PetscViewerGetAttachedVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, version));
357: if (*version) PetscFunctionReturn(PETSC_SUCCESS);
359: //TODO string HDF5 attribute handling is terrible and should be redesigned
360: PetscCall(PetscStrallocpy(DMPLEX_STORAGE_VERSION_FIRST, &defaultVersion));
361: PetscCall(PetscViewerHDF5ReadAttribute(viewer, "/", ATTR_NAME, PETSC_STRING, &defaultVersion, &versionString));
362: PetscCall(PetscViewerParseVersion_Private(viewer, versionString, version));
363: PetscCall(PetscViewerAttachVersion_Private(viewer, DMPLEX_STORAGE_VERSION_READING_KEY, *version));
364: PetscCall(PetscFree(versionString));
365: PetscCall(PetscFree(defaultVersion));
366: PetscFunctionReturn(PETSC_SUCCESS);
367: }
369: static PetscErrorCode DMPlexGetHDF5Name_Private(DM dm, const char *name[])
370: {
371: PetscFunctionBegin;
372: if (((PetscObject)dm)->name) {
373: PetscCall(PetscObjectGetName((PetscObject)dm, name));
374: } else {
375: *name = "plex";
376: }
377: PetscFunctionReturn(PETSC_SUCCESS);
378: }
380: PetscErrorCode DMSequenceGetLength_HDF5_Internal(DM dm, const char seqname[], PetscInt *seqlen, PetscViewer viewer)
381: {
382: hid_t file, group, dset, dspace;
383: hsize_t rdim, *dims;
384: char *groupname;
385: PetscBool has;
387: PetscFunctionBegin;
388: PetscCall(PetscViewerHDF5GetGroup(viewer, NULL, &groupname));
389: PetscCall(PetscViewerHDF5HasDataset(viewer, seqname, &has));
390: PetscCheck(has, PetscObjectComm((PetscObject)viewer), PETSC_ERR_FILE_UNEXPECTED, "Object (dataset) \"%s\" not stored in group %s", seqname, groupname);
392: PetscCall(PetscViewerHDF5OpenGroup(viewer, NULL, &file, &group));
393: PetscCallHDF5Return(dset, H5Dopen2, (group, seqname, H5P_DEFAULT));
394: PetscCallHDF5Return(dspace, H5Dget_space, (dset));
395: PetscCallHDF5ReturnNoCheck(rdim, H5Sget_simple_extent_dims, (dspace, NULL, NULL));
396: PetscCall(PetscMalloc1(rdim, &dims));
397: PetscCallHDF5ReturnNoCheck(rdim, H5Sget_simple_extent_dims, (dspace, dims, NULL));
398: *seqlen = (PetscInt)dims[0];
399: PetscCall(PetscFree(dims));
400: PetscCallHDF5(H5Dclose, (dset));
401: PetscCallHDF5(H5Gclose, (group));
402: PetscFunctionReturn(PETSC_SUCCESS);
403: }
405: static PetscErrorCode DMSequenceView_HDF5(DM dm, const char seqname[], PetscInt seqnum, PetscScalar value, PetscViewer viewer)
406: {
407: Vec stamp;
408: PetscMPIInt rank;
410: PetscFunctionBegin;
411: if (seqnum < 0) PetscFunctionReturn(PETSC_SUCCESS);
412: PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
413: PetscCall(VecCreateMPI(PetscObjectComm((PetscObject)viewer), rank ? 0 : 1, 1, &stamp));
414: PetscCall(VecSetBlockSize(stamp, 1));
415: PetscCall(PetscObjectSetName((PetscObject)stamp, seqname));
416: if (rank == 0) {
417: PetscReal timeScale;
418: PetscBool istime;
420: PetscCall(PetscStrncmp(seqname, "time", 5, &istime));
421: if (istime) {
422: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale));
423: value *= timeScale;
424: }
425: PetscCall(VecSetValue(stamp, 0, value, INSERT_VALUES));
426: }
427: PetscCall(VecAssemblyBegin(stamp));
428: PetscCall(VecAssemblyEnd(stamp));
429: PetscCall(PetscViewerHDF5PushGroup(viewer, "/"));
430: PetscCall(PetscViewerHDF5PushTimestepping(viewer));
431: PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum)); /* seqnum < 0 jumps out above */
432: PetscCall(VecView(stamp, viewer));
433: PetscCall(PetscViewerHDF5PopTimestepping(viewer));
434: PetscCall(PetscViewerHDF5PopGroup(viewer));
435: PetscCall(VecDestroy(&stamp));
436: PetscFunctionReturn(PETSC_SUCCESS);
437: }
439: PetscErrorCode DMSequenceLoad_HDF5_Internal(DM dm, const char seqname[], PetscInt seqnum, PetscScalar *value, PetscViewer viewer)
440: {
441: Vec stamp;
442: PetscMPIInt rank;
444: PetscFunctionBegin;
445: if (seqnum < 0) PetscFunctionReturn(PETSC_SUCCESS);
446: PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
447: PetscCall(VecCreateMPI(PetscObjectComm((PetscObject)viewer), rank ? 0 : 1, 1, &stamp));
448: PetscCall(VecSetBlockSize(stamp, 1));
449: PetscCall(PetscObjectSetName((PetscObject)stamp, seqname));
450: PetscCall(PetscViewerHDF5PushGroup(viewer, "/"));
451: PetscCall(PetscViewerHDF5PushTimestepping(viewer));
452: PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum)); /* seqnum < 0 jumps out above */
453: PetscCall(VecLoad(stamp, viewer));
454: PetscCall(PetscViewerHDF5PopTimestepping(viewer));
455: PetscCall(PetscViewerHDF5PopGroup(viewer));
456: if (rank == 0) {
457: const PetscScalar *a;
458: PetscReal timeScale;
459: PetscBool istime;
461: PetscCall(VecGetArrayRead(stamp, &a));
462: *value = a[0];
463: PetscCall(VecRestoreArrayRead(stamp, &a));
464: PetscCall(PetscStrncmp(seqname, "time", 5, &istime));
465: if (istime) {
466: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale));
467: *value /= timeScale;
468: }
469: }
470: PetscCall(VecDestroy(&stamp));
471: PetscFunctionReturn(PETSC_SUCCESS);
472: }
474: static PetscErrorCode DMPlexCreateCutVertexLabel_Private(DM dm, DMLabel cutLabel, DMLabel *cutVertexLabel)
475: {
476: IS cutcells = NULL;
477: const PetscInt *cutc;
478: PetscInt cellHeight, vStart, vEnd, cStart, cEnd, c;
480: PetscFunctionBegin;
481: if (!cutLabel) PetscFunctionReturn(PETSC_SUCCESS);
482: PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
483: PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
484: PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
485: /* Label vertices that should be duplicated */
486: PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Cut Vertices", cutVertexLabel));
487: PetscCall(DMLabelGetStratumIS(cutLabel, 2, &cutcells));
488: if (cutcells) {
489: PetscInt n;
491: PetscCall(ISGetIndices(cutcells, &cutc));
492: PetscCall(ISGetLocalSize(cutcells, &n));
493: for (c = 0; c < n; ++c) {
494: if ((cutc[c] >= cStart) && (cutc[c] < cEnd)) {
495: PetscInt *closure = NULL;
496: PetscInt closureSize, cl, value;
498: PetscCall(DMPlexGetTransitiveClosure(dm, cutc[c], PETSC_TRUE, &closureSize, &closure));
499: for (cl = 0; cl < closureSize * 2; cl += 2) {
500: if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) {
501: PetscCall(DMLabelGetValue(cutLabel, closure[cl], &value));
502: if (value == 1) PetscCall(DMLabelSetValue(*cutVertexLabel, closure[cl], 1));
503: }
504: }
505: PetscCall(DMPlexRestoreTransitiveClosure(dm, cutc[c], PETSC_TRUE, &closureSize, &closure));
506: }
507: }
508: PetscCall(ISRestoreIndices(cutcells, &cutc));
509: }
510: PetscCall(ISDestroy(&cutcells));
511: PetscFunctionReturn(PETSC_SUCCESS);
512: }
514: PetscErrorCode VecView_Plex_Local_HDF5_Internal(Vec v, PetscViewer viewer)
515: {
516: DM dm;
517: DM dmBC;
518: PetscSection section, sectionGlobal;
519: Vec gv;
520: const char *name;
521: PetscViewerFormat format;
522: PetscInt seqnum;
523: PetscReal seqval;
524: PetscBool isseq;
526: PetscFunctionBegin;
527: PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
528: PetscCall(VecGetDM(v, &dm));
529: PetscCall(DMGetLocalSection(dm, §ion));
530: PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, &seqval));
531: PetscCall(DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar)seqval, viewer));
532: if (seqnum >= 0) {
533: PetscCall(PetscViewerHDF5PushTimestepping(viewer));
534: PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
535: }
536: PetscCall(PetscViewerGetFormat(viewer, &format));
537: PetscCall(DMGetOutputDM(dm, &dmBC));
538: PetscCall(DMGetGlobalSection(dmBC, §ionGlobal));
539: PetscCall(DMGetGlobalVector(dmBC, &gv));
540: PetscCall(PetscObjectGetName((PetscObject)v, &name));
541: PetscCall(PetscObjectSetName((PetscObject)gv, name));
542: PetscCall(DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv));
543: PetscCall(DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv));
544: PetscCall(PetscObjectTypeCompare((PetscObject)gv, VECSEQ, &isseq));
545: if (format == PETSC_VIEWER_HDF5_VIZ) {
546: /* Output visualization representation */
547: PetscInt numFields, f;
548: DMLabel cutLabel, cutVertexLabel = NULL;
550: PetscCall(PetscSectionGetNumFields(section, &numFields));
551: PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
552: for (f = 0; f < numFields; ++f) {
553: Vec subv;
554: IS is;
555: const char *fname, *fgroup, *componentName, *fname_def = "unnamed";
556: char subname[PETSC_MAX_PATH_LEN];
557: PetscInt Nc, c;
558: PetscInt pStart, pEnd;
559: PetscViewerVTKFieldType ft;
561: PetscCall(DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft));
562: if (ft == PETSC_VTK_INVALID) continue;
563: fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
564: PetscCall(PetscSectionGetFieldName(section, f, &fname));
565: if (!fname) fname = fname_def;
567: PetscCall(PetscViewerHDF5PushGroup(viewer, fgroup));
569: if (cutLabel) {
570: const PetscScalar *ga;
571: PetscScalar *suba;
572: PetscInt gstart, subSize = 0, extSize = 0, subOff = 0, newOff = 0, p;
574: PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
575: PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
576: for (p = pStart; p < pEnd; ++p) {
577: PetscInt gdof, fdof = 0, val;
579: PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
580: if (gdof > 0) PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
581: subSize += fdof;
582: PetscCall(DMLabelGetValue(cutVertexLabel, p, &val));
583: if (val == 1) extSize += fdof;
584: }
585: PetscCall(VecCreate(PetscObjectComm((PetscObject)gv), &subv));
586: PetscCall(VecSetSizes(subv, subSize + extSize, PETSC_DETERMINE));
587: PetscCall(VecSetBlockSize(subv, Nc));
588: PetscCall(VecSetType(subv, VECSTANDARD));
589: PetscCall(VecGetOwnershipRange(gv, &gstart, NULL));
590: PetscCall(VecGetArrayRead(gv, &ga));
591: PetscCall(VecGetArray(subv, &suba));
592: for (p = pStart; p < pEnd; ++p) {
593: PetscInt gdof, goff, val;
595: PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
596: if (gdof > 0) {
597: PetscInt fdof, fc, f2, poff = 0;
599: PetscCall(PetscSectionGetOffset(sectionGlobal, p, &goff));
600: /* Can get rid of this loop by storing field information in the global section */
601: for (f2 = 0; f2 < f; ++f2) {
602: PetscCall(PetscSectionGetFieldDof(section, p, f2, &fdof));
603: poff += fdof;
604: }
605: PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
606: for (fc = 0; fc < fdof; ++fc, ++subOff) suba[subOff] = ga[goff + poff + fc - gstart];
607: PetscCall(DMLabelGetValue(cutVertexLabel, p, &val));
608: if (val == 1) {
609: for (fc = 0; fc < fdof; ++fc, ++newOff) suba[subSize + newOff] = ga[goff + poff + fc - gstart];
610: }
611: }
612: }
613: PetscCall(VecRestoreArrayRead(gv, &ga));
614: PetscCall(VecRestoreArray(subv, &suba));
615: PetscCall(DMLabelDestroy(&cutVertexLabel));
616: } else {
617: PetscCall(PetscSectionGetField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv));
618: }
619: PetscCall(PetscStrncpy(subname, name, sizeof(subname)));
620: PetscCall(PetscStrlcat(subname, "_", sizeof(subname)));
621: PetscCall(PetscStrlcat(subname, fname, sizeof(subname)));
622: PetscCall(PetscObjectSetName((PetscObject)subv, subname));
623: if (isseq) PetscCall(VecView_Seq(subv, viewer));
624: else PetscCall(VecView_MPI(subv, viewer));
625: if (ft == PETSC_VTK_POINT_VECTOR_FIELD || ft == PETSC_VTK_CELL_VECTOR_FIELD) {
626: PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, "vector_field_type", PETSC_STRING, "vector"));
627: } else {
628: PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, "vector_field_type", PETSC_STRING, "scalar"));
629: }
631: /* Output the component names in the field if available */
632: PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
633: for (c = 0; c < Nc; ++c) {
634: char componentNameLabel[PETSC_MAX_PATH_LEN];
635: PetscCall(PetscSectionGetComponentName(section, f, c, &componentName));
636: PetscCall(PetscSNPrintf(componentNameLabel, sizeof(componentNameLabel), "componentName%" PetscInt_FMT, c));
637: PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)subv, componentNameLabel, PETSC_STRING, componentName));
638: }
640: if (cutLabel) PetscCall(VecDestroy(&subv));
641: else PetscCall(PetscSectionRestoreField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv));
642: PetscCall(PetscViewerHDF5PopGroup(viewer));
643: }
644: } else {
645: /* Output full vector */
646: PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
647: if (isseq) PetscCall(VecView_Seq(gv, viewer));
648: else PetscCall(VecView_MPI(gv, viewer));
649: PetscCall(PetscViewerHDF5PopGroup(viewer));
650: }
651: if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
652: PetscCall(DMRestoreGlobalVector(dmBC, &gv));
653: PetscFunctionReturn(PETSC_SUCCESS);
654: }
656: PetscErrorCode VecView_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
657: {
658: DMPlexStorageVersion version;
659: DM dm;
660: Vec locv;
661: PetscObject isZero;
662: const char *name;
663: PetscReal time;
665: PetscFunctionBegin;
666: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
667: PetscCall(PetscInfo(v, "Writing Vec %s storage version %d.%d.%d\n", v->hdr.name, version->major, version->minor, version->subminor));
669: PetscCall(VecGetDM(v, &dm));
670: PetscCall(DMGetLocalVector(dm, &locv));
671: PetscCall(PetscObjectGetName((PetscObject)v, &name));
672: PetscCall(PetscObjectSetName((PetscObject)locv, name));
673: PetscCall(PetscObjectQuery((PetscObject)v, "__Vec_bc_zero__", &isZero));
674: PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", isZero));
675: PetscCall(DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv));
676: PetscCall(DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv));
677: PetscCall(DMGetOutputSequenceNumber(dm, NULL, &time));
678: PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL));
679: PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
680: if (DMPlexStorageVersionEQ(version, 1, 1, 0)) {
681: PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
682: PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ));
683: PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
684: PetscCall(PetscViewerPopFormat(viewer));
685: PetscCall(PetscViewerHDF5PopGroup(viewer));
686: }
687: PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", NULL));
688: PetscCall(DMRestoreLocalVector(dm, &locv));
689: PetscFunctionReturn(PETSC_SUCCESS);
690: }
692: PetscErrorCode VecView_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
693: {
694: PetscBool isseq;
696: PetscFunctionBegin;
697: PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
698: PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
699: if (isseq) PetscCall(VecView_Seq(v, viewer));
700: else PetscCall(VecView_MPI(v, viewer));
701: PetscCall(PetscViewerHDF5PopGroup(viewer));
702: PetscFunctionReturn(PETSC_SUCCESS);
703: }
705: PetscErrorCode VecLoad_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
706: {
707: DM dm;
708: Vec locv;
709: const char *name;
710: PetscInt seqnum;
712: PetscFunctionBegin;
713: PetscCall(VecGetDM(v, &dm));
714: PetscCall(DMGetLocalVector(dm, &locv));
715: PetscCall(PetscObjectGetName((PetscObject)v, &name));
716: PetscCall(PetscObjectSetName((PetscObject)locv, name));
717: PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, NULL));
718: PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
719: if (seqnum >= 0) {
720: PetscCall(PetscViewerHDF5PushTimestepping(viewer));
721: PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
722: }
723: PetscCall(VecLoad_Plex_Local(locv, viewer));
724: if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
725: PetscCall(PetscViewerHDF5PopGroup(viewer));
726: PetscCall(DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v));
727: PetscCall(DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v));
728: PetscCall(DMRestoreLocalVector(dm, &locv));
729: PetscFunctionReturn(PETSC_SUCCESS);
730: }
732: PetscErrorCode VecLoad_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
733: {
734: DM dm;
735: PetscInt seqnum;
737: PetscFunctionBegin;
738: PetscCall(VecGetDM(v, &dm));
739: PetscCall(DMGetOutputSequenceNumber(dm, &seqnum, NULL));
740: PetscCall(PetscViewerHDF5PushGroup(viewer, "/fields"));
741: if (seqnum >= 0) {
742: PetscCall(PetscViewerHDF5PushTimestepping(viewer));
743: PetscCall(PetscViewerHDF5SetTimestep(viewer, seqnum));
744: }
745: PetscCall(VecLoad_Default(v, viewer));
746: if (seqnum >= 0) PetscCall(PetscViewerHDF5PopTimestepping(viewer));
747: PetscCall(PetscViewerHDF5PopGroup(viewer));
748: PetscFunctionReturn(PETSC_SUCCESS);
749: }
751: static PetscErrorCode DMPlexDistributionView_HDF5_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
752: {
753: MPI_Comm comm;
754: PetscMPIInt size, rank;
755: PetscInt size_petsc_int;
756: const char *topologydm_name, *distribution_name;
757: const PetscInt *gpoint;
758: PetscInt pStart, pEnd, p;
759: PetscSF pointSF;
760: PetscInt nroots, nleaves;
761: const PetscInt *ilocal;
762: const PetscSFNode *iremote;
763: IS chartSizesIS, ownersIS, gpointsIS;
764: PetscInt *chartSize, *owners, *gpoints;
766: PetscFunctionBegin;
767: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
768: PetscCallMPI(MPI_Comm_size(comm, &size));
769: PetscCallMPI(MPI_Comm_rank(comm, &rank));
770: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
771: PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
772: if (!distribution_name) PetscFunctionReturn(PETSC_SUCCESS);
773: PetscCall(PetscLogEventBegin(DMPLEX_DistributionView, viewer, 0, 0, 0));
774: size_petsc_int = (PetscInt)size;
775: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "comm_size", PETSC_INT, (void *)&size_petsc_int));
776: PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
777: PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
778: PetscCall(PetscMalloc1(1, &chartSize));
779: *chartSize = pEnd - pStart;
780: PetscCall(PetscMalloc1(*chartSize, &owners));
781: PetscCall(PetscMalloc1(*chartSize, &gpoints));
782: PetscCall(DMGetPointSF(dm, &pointSF));
783: PetscCall(PetscSFGetGraph(pointSF, &nroots, &nleaves, &ilocal, &iremote));
784: for (p = pStart; p < pEnd; ++p) {
785: PetscInt gp = gpoint[p - pStart];
787: owners[p - pStart] = rank;
788: gpoints[p - pStart] = (gp < 0 ? -(gp + 1) : gp);
789: }
790: for (p = 0; p < nleaves; ++p) {
791: PetscInt ilocalp = (ilocal ? ilocal[p] : p);
793: owners[ilocalp] = iremote[p].rank;
794: }
795: PetscCall(ISCreateGeneral(comm, 1, chartSize, PETSC_OWN_POINTER, &chartSizesIS));
796: PetscCall(ISCreateGeneral(comm, *chartSize, owners, PETSC_OWN_POINTER, &ownersIS));
797: PetscCall(ISCreateGeneral(comm, *chartSize, gpoints, PETSC_OWN_POINTER, &gpointsIS));
798: PetscCall(PetscObjectSetName((PetscObject)chartSizesIS, "chart_sizes"));
799: PetscCall(PetscObjectSetName((PetscObject)ownersIS, "owners"));
800: PetscCall(PetscObjectSetName((PetscObject)gpointsIS, "global_point_numbers"));
801: PetscCall(ISView(chartSizesIS, viewer));
802: PetscCall(ISView(ownersIS, viewer));
803: PetscCall(ISView(gpointsIS, viewer));
804: PetscCall(ISDestroy(&chartSizesIS));
805: PetscCall(ISDestroy(&ownersIS));
806: PetscCall(ISDestroy(&gpointsIS));
807: PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
808: PetscCall(PetscLogEventEnd(DMPLEX_DistributionView, viewer, 0, 0, 0));
809: PetscFunctionReturn(PETSC_SUCCESS);
810: }
812: static PetscErrorCode DMPlexTopologyView_HDF5_Inner_Private(DM dm, IS globalPointNumbers, PetscViewer viewer, PetscInt pStart, PetscInt pEnd, const char pointsName[], const char coneSizesName[], const char conesName[], const char orientationsName[])
813: {
814: IS coneSizesIS, conesIS, orientationsIS;
815: PetscInt *coneSizes, *cones, *orientations;
816: const PetscInt *gpoint;
817: PetscInt nPoints = 0, conesSize = 0;
818: PetscInt p, c, s;
819: MPI_Comm comm;
821: PetscFunctionBegin;
822: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
823: PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
824: for (p = pStart; p < pEnd; ++p) {
825: if (gpoint[p] >= 0) {
826: PetscInt coneSize;
828: PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
829: nPoints += 1;
830: conesSize += coneSize;
831: }
832: }
833: PetscCall(PetscMalloc1(nPoints, &coneSizes));
834: PetscCall(PetscMalloc1(conesSize, &cones));
835: PetscCall(PetscMalloc1(conesSize, &orientations));
836: for (p = pStart, c = 0, s = 0; p < pEnd; ++p) {
837: if (gpoint[p] >= 0) {
838: const PetscInt *cone, *ornt;
839: PetscInt coneSize, cp;
841: PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
842: PetscCall(DMPlexGetCone(dm, p, &cone));
843: PetscCall(DMPlexGetConeOrientation(dm, p, &ornt));
844: coneSizes[s] = coneSize;
845: for (cp = 0; cp < coneSize; ++cp, ++c) {
846: cones[c] = gpoint[cone[cp]] < 0 ? -(gpoint[cone[cp]] + 1) : gpoint[cone[cp]];
847: orientations[c] = ornt[cp];
848: }
849: ++s;
850: }
851: }
852: PetscCheck(s == nPoints, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of points %" PetscInt_FMT " != %" PetscInt_FMT, s, nPoints);
853: PetscCheck(c == conesSize, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cone points %" PetscInt_FMT " != %" PetscInt_FMT, c, conesSize);
854: PetscCall(ISCreateGeneral(comm, nPoints, coneSizes, PETSC_OWN_POINTER, &coneSizesIS));
855: PetscCall(ISCreateGeneral(comm, conesSize, cones, PETSC_OWN_POINTER, &conesIS));
856: PetscCall(ISCreateGeneral(comm, conesSize, orientations, PETSC_OWN_POINTER, &orientationsIS));
857: PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
858: PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
859: PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
860: PetscCall(ISView(coneSizesIS, viewer));
861: PetscCall(ISView(conesIS, viewer));
862: PetscCall(ISView(orientationsIS, viewer));
863: PetscCall(ISDestroy(&coneSizesIS));
864: PetscCall(ISDestroy(&conesIS));
865: PetscCall(ISDestroy(&orientationsIS));
866: if (pointsName) {
867: IS pointsIS;
868: PetscInt *points;
870: PetscCall(PetscMalloc1(nPoints, &points));
871: for (p = pStart, c = 0, s = 0; p < pEnd; ++p) {
872: if (gpoint[p] >= 0) {
873: points[s] = gpoint[p];
874: ++s;
875: }
876: }
877: PetscCall(ISCreateGeneral(comm, nPoints, points, PETSC_OWN_POINTER, &pointsIS));
878: PetscCall(PetscObjectSetName((PetscObject)pointsIS, pointsName));
879: PetscCall(ISView(pointsIS, viewer));
880: PetscCall(ISDestroy(&pointsIS));
881: }
882: PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
883: PetscFunctionReturn(PETSC_SUCCESS);
884: }
886: static PetscErrorCode DMPlexTopologyView_HDF5_Legacy_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
887: {
888: const char *pointsName, *coneSizesName, *conesName, *orientationsName;
889: PetscInt pStart, pEnd, dim;
891: PetscFunctionBegin;
892: pointsName = "order";
893: coneSizesName = "cones";
894: conesName = "cells";
895: orientationsName = "orientation";
896: PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
897: PetscCall(DMPlexTopologyView_HDF5_Inner_Private(dm, globalPointNumbers, viewer, pStart, pEnd, pointsName, coneSizesName, conesName, orientationsName));
898: PetscCall(DMGetDimension(dm, &dim));
899: PetscCall(PetscViewerHDF5WriteAttribute(viewer, conesName, "cell_dim", PETSC_INT, (void *)&dim));
900: PetscFunctionReturn(PETSC_SUCCESS);
901: }
903: //TODO get this numbering right away without needing this function
904: /* Renumber global point numbers so that they are 0-based per stratum */
905: static PetscErrorCode RenumberGlobalPointNumbersPerStratum_Private(DM dm, IS globalPointNumbers, IS *newGlobalPointNumbers, IS *strataPermutation)
906: {
907: PetscInt d, depth, p, n;
908: PetscInt *offsets;
909: const PetscInt *gpn;
910: PetscInt *ngpn;
911: MPI_Comm comm;
913: PetscFunctionBegin;
914: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
915: PetscCall(ISGetLocalSize(globalPointNumbers, &n));
916: PetscCall(ISGetIndices(globalPointNumbers, &gpn));
917: PetscCall(PetscMalloc1(n, &ngpn));
918: PetscCall(DMPlexGetDepth(dm, &depth));
919: PetscCall(PetscMalloc1(depth + 1, &offsets));
920: for (d = 0; d <= depth; d++) {
921: PetscInt pStart, pEnd;
923: PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
924: offsets[d] = PETSC_INT_MAX;
925: for (p = pStart; p < pEnd; p++) {
926: if (gpn[p] >= 0 && gpn[p] < offsets[d]) offsets[d] = gpn[p];
927: }
928: }
929: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, offsets, depth + 1, MPIU_INT, MPI_MIN, comm));
930: for (d = 0; d <= depth; d++) {
931: PetscInt pStart, pEnd;
933: PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
934: for (p = pStart; p < pEnd; p++) ngpn[p] = gpn[p] - PetscSign(gpn[p]) * offsets[d];
935: }
936: PetscCall(ISRestoreIndices(globalPointNumbers, &gpn));
937: PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)globalPointNumbers), n, ngpn, PETSC_OWN_POINTER, newGlobalPointNumbers));
938: {
939: PetscInt *perm;
941: PetscCall(PetscMalloc1(depth + 1, &perm));
942: for (d = 0; d <= depth; d++) perm[d] = d;
943: PetscCall(PetscSortIntWithPermutation(depth + 1, offsets, perm));
944: PetscCall(ISCreateGeneral(PETSC_COMM_SELF, depth + 1, perm, PETSC_OWN_POINTER, strataPermutation));
945: }
946: PetscCall(PetscFree(offsets));
947: PetscFunctionReturn(PETSC_SUCCESS);
948: }
950: static PetscErrorCode DMPlexTopologyView_HDF5_Private(DM dm, IS globalPointNumbers, PetscViewer viewer)
951: {
952: IS globalPointNumbers0, strataPermutation;
953: const char *coneSizesName, *conesName, *orientationsName;
954: PetscInt depth, d;
955: MPI_Comm comm;
957: PetscFunctionBegin;
958: coneSizesName = "cone_sizes";
959: conesName = "cones";
960: orientationsName = "orientations";
961: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
962: PetscCall(DMPlexGetDepth(dm, &depth));
963: {
964: PetscInt dim;
966: PetscCall(DMGetDimension(dm, &dim));
967: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "cell_dim", PETSC_INT, &dim));
968: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "depth", PETSC_INT, &depth));
969: }
971: PetscCall(RenumberGlobalPointNumbersPerStratum_Private(dm, globalPointNumbers, &globalPointNumbers0, &strataPermutation));
972: /* TODO dirty trick to save serial IS using the same parallel viewer */
973: {
974: IS spOnComm;
975: PetscInt n = 0, N;
976: const PetscInt *idx = NULL;
977: const PetscInt *old;
978: PetscMPIInt rank;
980: PetscCallMPI(MPI_Comm_rank(comm, &rank));
981: PetscCall(ISGetLocalSize(strataPermutation, &N));
982: PetscCall(ISGetIndices(strataPermutation, &old));
983: if (!rank) {
984: n = N;
985: idx = old;
986: }
987: PetscCall(ISCreateGeneral(comm, n, idx, PETSC_COPY_VALUES, &spOnComm));
988: PetscCall(ISRestoreIndices(strataPermutation, &old));
989: PetscCall(ISDestroy(&strataPermutation));
990: strataPermutation = spOnComm;
991: }
992: PetscCall(PetscObjectSetName((PetscObject)strataPermutation, "permutation"));
993: PetscCall(ISView(strataPermutation, viewer));
994: PetscCall(PetscViewerHDF5PushGroup(viewer, "strata"));
995: for (d = 0; d <= depth; d++) {
996: PetscInt pStart, pEnd;
997: char group[128];
999: PetscCall(PetscSNPrintf(group, sizeof(group), "%" PetscInt_FMT, d));
1000: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1001: PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
1002: PetscCall(DMPlexTopologyView_HDF5_Inner_Private(dm, globalPointNumbers0, viewer, pStart, pEnd, NULL, coneSizesName, conesName, orientationsName));
1003: PetscCall(PetscViewerHDF5PopGroup(viewer));
1004: }
1005: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* strata */
1006: PetscCall(ISDestroy(&globalPointNumbers0));
1007: PetscCall(ISDestroy(&strataPermutation));
1008: PetscFunctionReturn(PETSC_SUCCESS);
1009: }
1011: PetscErrorCode DMPlexTopologyView_HDF5_Internal(DM dm, IS globalPointNumbers, PetscViewer viewer)
1012: {
1013: DMPlexStorageVersion version;
1014: const char *topologydm_name;
1015: char group[PETSC_MAX_PATH_LEN];
1017: PetscFunctionBegin;
1018: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1019: PetscCall(PetscInfo(dm, "Writing DM %s storage version %d.%d.%d\n", dm->hdr.name, version->major, version->minor, version->subminor));
1020: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1021: if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1022: PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s", topologydm_name));
1023: } else {
1024: PetscCall(PetscStrncpy(group, "/", sizeof(group)));
1025: }
1026: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1028: PetscCall(PetscViewerHDF5PushGroup(viewer, "topology"));
1029: if (version->major < 3) {
1030: PetscCall(DMPlexTopologyView_HDF5_Legacy_Private(dm, globalPointNumbers, viewer));
1031: } else {
1032: /* since DMPlexStorageVersion 3.0.0 */
1033: PetscCall(DMPlexTopologyView_HDF5_Private(dm, globalPointNumbers, viewer));
1034: }
1035: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "topology" */
1037: if (DMPlexStorageVersionGE(version, 2, 1, 0)) {
1038: const char *distribution_name;
1040: PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
1041: PetscCall(PetscViewerHDF5PushGroup(viewer, "distributions"));
1042: PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));
1043: PetscCall(PetscViewerHDF5PushGroup(viewer, distribution_name));
1044: PetscCall(DMPlexDistributionView_HDF5_Private(dm, globalPointNumbers, viewer));
1045: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* distribution_name */
1046: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "distributions" */
1047: }
1049: PetscCall(PetscViewerHDF5PopGroup(viewer));
1050: PetscFunctionReturn(PETSC_SUCCESS);
1051: }
1053: static PetscErrorCode CreateConesIS_Private(DM dm, PetscInt cStart, PetscInt cEnd, IS globalCellNumbers, PetscInt *numCorners, IS *cellIS)
1054: {
1055: PetscSF sfPoint;
1056: DMLabel cutLabel, cutVertexLabel = NULL;
1057: IS globalVertexNumbers, cutvertices = NULL;
1058: const PetscInt *gcell, *gvertex, *cutverts = NULL;
1059: PetscInt *vertices;
1060: PetscInt conesSize = 0;
1061: PetscInt dim, numCornersLocal = 0, cell, vStart, vEnd, vExtra = 0, v;
1063: PetscFunctionBegin;
1064: *numCorners = 0;
1065: PetscCall(DMGetDimension(dm, &dim));
1066: PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1067: PetscCall(ISGetIndices(globalCellNumbers, &gcell));
1069: for (cell = cStart; cell < cEnd; ++cell) {
1070: PetscInt *closure = NULL;
1071: PetscInt closureSize, v, Nc = 0;
1073: if (gcell[cell] < 0) continue;
1074: PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1075: for (v = 0; v < closureSize * 2; v += 2) {
1076: if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
1077: }
1078: PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1079: conesSize += Nc;
1080: if (!numCornersLocal) numCornersLocal = Nc;
1081: else if (numCornersLocal != Nc) numCornersLocal = 1;
1082: }
1083: PetscCallMPI(MPIU_Allreduce(&numCornersLocal, numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
1084: PetscCheck(!numCornersLocal || !(numCornersLocal != *numCorners || *numCorners == 0), PETSC_COMM_SELF, PETSC_ERR_SUP, "Visualization topology currently only supports identical cell shapes");
1085: /* Handle periodic cuts by identifying vertices which should be duplicated */
1086: PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1087: PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
1088: if (cutVertexLabel) PetscCall(DMLabelGetStratumIS(cutVertexLabel, 1, &cutvertices));
1089: if (cutvertices) {
1090: PetscCall(ISGetIndices(cutvertices, &cutverts));
1091: PetscCall(ISGetLocalSize(cutvertices, &vExtra));
1092: }
1093: PetscCall(DMGetPointSF(dm, &sfPoint));
1094: if (cutLabel) {
1095: const PetscInt *ilocal;
1096: const PetscSFNode *iremote;
1097: PetscInt nroots, nleaves;
1099: PetscCall(PetscSFGetGraph(sfPoint, &nroots, &nleaves, &ilocal, &iremote));
1100: if (nleaves < 0) {
1101: PetscCall(PetscObjectReference((PetscObject)sfPoint));
1102: } else {
1103: PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)sfPoint), &sfPoint));
1104: PetscCall(PetscSFSetGraph(sfPoint, nroots + vExtra, nleaves, (PetscInt *)ilocal, PETSC_COPY_VALUES, (PetscSFNode *)iremote, PETSC_COPY_VALUES));
1105: }
1106: } else {
1107: PetscCall(PetscObjectReference((PetscObject)sfPoint));
1108: }
1109: /* Number all vertices */
1110: PetscCall(DMPlexCreateNumbering_Plex(dm, vStart, vEnd + vExtra, 0, NULL, sfPoint, &globalVertexNumbers));
1111: PetscCall(PetscSFDestroy(&sfPoint));
1112: /* Create cones */
1113: PetscCall(ISGetIndices(globalVertexNumbers, &gvertex));
1114: PetscCall(PetscMalloc1(conesSize, &vertices));
1115: for (cell = cStart, v = 0; cell < cEnd; ++cell) {
1116: PetscInt *closure = NULL;
1117: PetscInt closureSize, Nc = 0, p, value = -1;
1118: PetscBool replace;
1120: if (gcell[cell] < 0) continue;
1121: if (cutLabel) PetscCall(DMLabelGetValue(cutLabel, cell, &value));
1122: replace = (value == 2) ? PETSC_TRUE : PETSC_FALSE;
1123: PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1124: for (p = 0; p < closureSize * 2; p += 2) {
1125: if ((closure[p] >= vStart) && (closure[p] < vEnd)) closure[Nc++] = closure[p];
1126: }
1127: PetscCall(DMPlexReorderCell(dm, cell, closure));
1128: for (p = 0; p < Nc; ++p) {
1129: PetscInt nv, gv = gvertex[closure[p] - vStart];
1131: if (replace) {
1132: PetscCall(PetscFindInt(closure[p], vExtra, cutverts, &nv));
1133: if (nv >= 0) gv = gvertex[vEnd - vStart + nv];
1134: }
1135: vertices[v++] = gv < 0 ? -(gv + 1) : gv;
1136: }
1137: PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1138: }
1139: PetscCall(ISRestoreIndices(globalVertexNumbers, &gvertex));
1140: PetscCall(ISDestroy(&globalVertexNumbers));
1141: PetscCall(ISRestoreIndices(globalCellNumbers, &gcell));
1142: if (cutvertices) PetscCall(ISRestoreIndices(cutvertices, &cutverts));
1143: PetscCall(ISDestroy(&cutvertices));
1144: PetscCall(DMLabelDestroy(&cutVertexLabel));
1145: PetscCheck(v == conesSize, PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %" PetscInt_FMT " != %" PetscInt_FMT, v, conesSize);
1146: PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), conesSize, vertices, PETSC_OWN_POINTER, cellIS));
1147: PetscCall(PetscLayoutSetBlockSize((*cellIS)->map, *numCorners));
1148: PetscCall(PetscObjectSetName((PetscObject)*cellIS, "cells"));
1149: PetscFunctionReturn(PETSC_SUCCESS);
1150: }
1152: static PetscErrorCode DMPlexTopologyView_HDF5_XDMF_Private(DM dm, IS globalCellNumbers, PetscViewer viewer)
1153: {
1154: DM cdm;
1155: DMLabel depthLabel, ctLabel;
1156: IS cellIS;
1157: PetscInt dim, depth, cellHeight, c, n = 0;
1159: PetscFunctionBegin;
1160: PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz"));
1161: PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));
1163: PetscCall(PetscViewerHDF5PopGroup(viewer));
1164: PetscCall(DMGetDimension(dm, &dim));
1165: PetscCall(DMPlexGetDepth(dm, &depth));
1166: PetscCall(DMGetCoordinateDM(dm, &cdm));
1167: PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
1168: PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
1169: PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
1170: for (c = 0; c < DM_NUM_POLYTOPES; ++c) {
1171: const DMPolytopeType ict = (DMPolytopeType)c;
1172: PetscInt pStart, pEnd, dep, numCorners;
1173: PetscBool output = PETSC_FALSE, doOutput;
1175: if (ict == DM_POLYTOPE_FV_GHOST) continue;
1176: PetscCall(DMLabelGetStratumBounds(ctLabel, ict, &pStart, &pEnd));
1177: if (pStart >= 0) {
1178: PetscCall(DMLabelGetValue(depthLabel, pStart, &dep));
1179: if (dep == depth - cellHeight) output = PETSC_TRUE;
1180: }
1181: PetscCallMPI(MPIU_Allreduce(&output, &doOutput, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
1182: if (!doOutput) continue;
1183: PetscCall(CreateConesIS_Private(dm, pStart, pEnd, globalCellNumbers, &numCorners, &cellIS));
1184: if (!n) {
1185: PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz/topology"));
1186: } else {
1187: char group[PETSC_MAX_PATH_LEN];
1189: PetscCall(PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/viz/topology_%" PetscInt_FMT, n));
1190: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1191: }
1192: PetscCall(ISView(cellIS, viewer));
1193: PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)cellIS, "cell_corners", PETSC_INT, (void *)&numCorners));
1194: PetscCall(PetscViewerHDF5WriteObjectAttribute(viewer, (PetscObject)cellIS, "cell_dim", PETSC_INT, (void *)&dim));
1195: PetscCall(ISDestroy(&cellIS));
1196: PetscCall(PetscViewerHDF5PopGroup(viewer));
1197: ++n;
1198: }
1199: PetscFunctionReturn(PETSC_SUCCESS);
1200: }
1202: static PetscErrorCode DMPlexCoordinatesView_HDF5_Legacy_Private(DM dm, PetscViewer viewer)
1203: {
1204: DM cdm;
1205: Vec coordinates, newcoords;
1206: PetscReal lengthScale;
1207: PetscInt m, M, bs;
1209: PetscFunctionBegin;
1210: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1211: PetscCall(DMGetCoordinateDM(dm, &cdm));
1212: PetscCall(DMGetCoordinates(dm, &coordinates));
1213: PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &newcoords));
1214: PetscCall(PetscObjectSetName((PetscObject)newcoords, "vertices"));
1215: PetscCall(VecGetSize(coordinates, &M));
1216: PetscCall(VecGetLocalSize(coordinates, &m));
1217: PetscCall(VecSetSizes(newcoords, m, M));
1218: PetscCall(VecGetBlockSize(coordinates, &bs));
1219: PetscCall(VecSetBlockSize(newcoords, bs));
1220: PetscCall(VecSetType(newcoords, VECSTANDARD));
1221: PetscCall(VecCopy(coordinates, newcoords));
1222: PetscCall(VecScale(newcoords, lengthScale));
1223: /* Did not use DMGetGlobalVector() in order to bypass default group assignment */
1224: PetscCall(PetscViewerHDF5PushGroup(viewer, "/geometry"));
1225: PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
1226: PetscCall(VecView(newcoords, viewer));
1227: PetscCall(PetscViewerPopFormat(viewer));
1228: PetscCall(PetscViewerHDF5PopGroup(viewer));
1229: PetscCall(VecDestroy(&newcoords));
1230: PetscFunctionReturn(PETSC_SUCCESS);
1231: }
1233: PetscErrorCode DMPlexCoordinatesView_HDF5_Internal(DM dm, PetscViewer viewer)
1234: {
1235: DM cdm;
1236: Vec coords, newcoords;
1237: PetscInt m, M, bs;
1238: PetscReal lengthScale;
1239: PetscBool viewSection = PETSC_TRUE;
1240: const char *topologydm_name, *coordinatedm_name, *coordinates_name;
1242: PetscFunctionBegin;
1243: {
1244: PetscViewerFormat format;
1245: DMPlexStorageVersion version;
1247: PetscCall(PetscViewerGetFormat(viewer, &format));
1248: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1249: if (!DMPlexStorageVersionGE(version, 2, 0, 0) || format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1250: PetscCall(DMPlexCoordinatesView_HDF5_Legacy_Private(dm, viewer));
1251: PetscFunctionReturn(PETSC_SUCCESS);
1252: }
1253: }
1254: /* since 2.0.0 */
1255: PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_view_coordinate_section", &viewSection, NULL));
1256: PetscCall(DMGetCoordinateDM(dm, &cdm));
1257: PetscCall(DMGetCoordinates(dm, &coords));
1258: PetscCall(PetscObjectGetName((PetscObject)cdm, &coordinatedm_name));
1259: PetscCall(PetscObjectGetName((PetscObject)coords, &coordinates_name));
1260: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1261: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1262: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1263: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "coordinateDMName", PETSC_STRING, coordinatedm_name));
1264: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "coordinatesName", PETSC_STRING, coordinates_name));
1265: PetscCall(PetscViewerHDF5PopGroup(viewer));
1266: PetscCall(PetscViewerHDF5PopGroup(viewer));
1267: if (viewSection) PetscCall(DMPlexSectionView(dm, viewer, cdm));
1268: PetscCall(VecCreate(PetscObjectComm((PetscObject)coords), &newcoords));
1269: PetscCall(PetscObjectSetName((PetscObject)newcoords, coordinates_name));
1270: PetscCall(VecGetSize(coords, &M));
1271: PetscCall(VecGetLocalSize(coords, &m));
1272: PetscCall(VecSetSizes(newcoords, m, M));
1273: PetscCall(VecGetBlockSize(coords, &bs));
1274: PetscCall(VecSetBlockSize(newcoords, bs));
1275: PetscCall(VecSetType(newcoords, VECSTANDARD));
1276: PetscCall(VecCopy(coords, newcoords));
1277: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1278: PetscCall(VecScale(newcoords, lengthScale));
1279: PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
1280: PetscCall(DMPlexGlobalVectorView(dm, viewer, cdm, newcoords));
1281: PetscCall(PetscViewerPopFormat(viewer));
1282: PetscCall(VecDestroy(&newcoords));
1283: PetscFunctionReturn(PETSC_SUCCESS);
1284: }
1286: static PetscErrorCode DMPlexCoordinatesView_HDF5_XDMF_Private(DM dm, PetscViewer viewer)
1287: {
1288: DM cdm;
1289: Vec coordinatesLocal, newcoords;
1290: PetscSection cSection, cGlobalSection;
1291: PetscScalar *coords, *ncoords;
1292: DMLabel cutLabel, cutVertexLabel = NULL;
1293: const PetscReal *L;
1294: PetscReal lengthScale;
1295: PetscInt vStart, vEnd, v, bs, N, coordSize, dof, off, d;
1296: PetscBool localized, embedded;
1298: PetscFunctionBegin;
1299: PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1300: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
1301: PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
1302: PetscCall(VecGetBlockSize(coordinatesLocal, &bs));
1303: PetscCall(DMGetCoordinatesLocalized(dm, &localized));
1304: if (localized == PETSC_FALSE) PetscFunctionReturn(PETSC_SUCCESS);
1305: PetscCall(DMGetPeriodicity(dm, NULL, NULL, &L));
1306: PetscCall(DMGetCoordinateDM(dm, &cdm));
1307: PetscCall(DMGetLocalSection(cdm, &cSection));
1308: PetscCall(DMGetGlobalSection(cdm, &cGlobalSection));
1309: PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1310: N = 0;
1312: PetscCall(DMPlexCreateCutVertexLabel_Private(dm, cutLabel, &cutVertexLabel));
1313: PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &newcoords));
1314: PetscCall(PetscSectionGetDof(cSection, vStart, &dof));
1315: PetscCall(PetscPrintf(PETSC_COMM_SELF, "DOF: %" PetscInt_FMT "\n", dof));
1316: embedded = (PetscBool)(L && dof == 2 && !cutLabel);
1317: if (cutVertexLabel) {
1318: PetscCall(DMLabelGetStratumSize(cutVertexLabel, 1, &v));
1319: N += dof * v;
1320: }
1321: for (v = vStart; v < vEnd; ++v) {
1322: PetscCall(PetscSectionGetDof(cGlobalSection, v, &dof));
1323: if (dof < 0) continue;
1324: if (embedded) N += dof + 1;
1325: else N += dof;
1326: }
1327: if (embedded) PetscCall(VecSetBlockSize(newcoords, bs + 1));
1328: else PetscCall(VecSetBlockSize(newcoords, bs));
1329: PetscCall(VecSetSizes(newcoords, N, PETSC_DETERMINE));
1330: PetscCall(VecSetType(newcoords, VECSTANDARD));
1331: PetscCall(VecGetArray(coordinatesLocal, &coords));
1332: PetscCall(VecGetArray(newcoords, &ncoords));
1333: coordSize = 0;
1334: for (v = vStart; v < vEnd; ++v) {
1335: PetscCall(PetscSectionGetDof(cGlobalSection, v, &dof));
1336: PetscCall(PetscSectionGetOffset(cSection, v, &off));
1337: if (dof < 0) continue;
1338: if (embedded) {
1339: if (L && (L[0] > 0.0) && (L[1] > 0.0)) {
1340: PetscReal theta, phi, r, R;
1341: /* XY-periodic */
1342: /* Suppose its an y-z circle, then
1343: \hat r = (0, cos(th), sin(th)) \hat x = (1, 0, 0)
1344: and the circle in that plane is
1345: \hat r cos(phi) + \hat x sin(phi) */
1346: theta = 2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1];
1347: phi = 2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0];
1348: r = L[0] / (2.0 * PETSC_PI * 2.0 * L[1]);
1349: R = L[1] / (2.0 * PETSC_PI);
1350: ncoords[coordSize++] = PetscSinReal(phi) * r;
1351: ncoords[coordSize++] = -PetscCosReal(theta) * (R + r * PetscCosReal(phi));
1352: ncoords[coordSize++] = PetscSinReal(theta) * (R + r * PetscCosReal(phi));
1353: } else if (L && (L[0] > 0.0)) {
1354: /* X-periodic */
1355: ncoords[coordSize++] = -PetscCosReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0]) * (L[0] / (2.0 * PETSC_PI));
1356: ncoords[coordSize++] = coords[off + 1];
1357: ncoords[coordSize++] = PetscSinReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 0]) / L[0]) * (L[0] / (2.0 * PETSC_PI));
1358: } else if (L && (L[1] > 0.0)) {
1359: /* Y-periodic */
1360: ncoords[coordSize++] = coords[off + 0];
1361: ncoords[coordSize++] = PetscSinReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1]) * (L[1] / (2.0 * PETSC_PI));
1362: ncoords[coordSize++] = -PetscCosReal(2.0 * PETSC_PI * PetscRealPart(coords[off + 1]) / L[1]) * (L[1] / (2.0 * PETSC_PI));
1363: #if 0
1364: } else if ((bd[0] == DM_BOUNDARY_TWIST)) {
1365: PetscReal phi, r, R;
1366: /* Mobius strip */
1367: /* Suppose its an x-z circle, then
1368: \hat r = (-cos(phi), 0, sin(phi)) \hat y = (0, 1, 0)
1369: and in that plane we rotate by pi as we go around the circle
1370: \hat r cos(phi/2) + \hat y sin(phi/2) */
1371: phi = 2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0];
1372: R = L[0];
1373: r = PetscRealPart(coords[off+1]) - L[1]/2.0;
1374: ncoords[coordSize++] = -PetscCosReal(phi) * (R + r * PetscCosReal(phi/2.0));
1375: ncoords[coordSize++] = PetscSinReal(phi/2.0) * r;
1376: ncoords[coordSize++] = PetscSinReal(phi) * (R + r * PetscCosReal(phi/2.0));
1377: #endif
1378: } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Cannot handle periodicity in this domain");
1379: } else {
1380: for (d = 0; d < dof; ++d, ++coordSize) ncoords[coordSize] = coords[off + d];
1381: }
1382: }
1383: if (cutVertexLabel) {
1384: IS vertices;
1385: const PetscInt *verts;
1386: PetscInt n;
1388: PetscCall(DMLabelGetStratumIS(cutVertexLabel, 1, &vertices));
1389: if (vertices) {
1390: PetscCall(ISGetIndices(vertices, &verts));
1391: PetscCall(ISGetLocalSize(vertices, &n));
1392: for (v = 0; v < n; ++v) {
1393: PetscCall(PetscSectionGetDof(cSection, verts[v], &dof));
1394: PetscCall(PetscSectionGetOffset(cSection, verts[v], &off));
1395: for (d = 0; d < dof; ++d) ncoords[coordSize++] = coords[off + d] + ((L[d] > 0.) ? L[d] : 0.0);
1396: }
1397: PetscCall(ISRestoreIndices(vertices, &verts));
1398: PetscCall(ISDestroy(&vertices));
1399: }
1400: }
1401: PetscCheck(coordSize == N, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatched sizes: %" PetscInt_FMT " != %" PetscInt_FMT, coordSize, N);
1402: PetscCall(DMLabelDestroy(&cutVertexLabel));
1403: PetscCall(VecRestoreArray(coordinatesLocal, &coords));
1404: PetscCall(VecRestoreArray(newcoords, &ncoords));
1405: PetscCall(PetscObjectSetName((PetscObject)newcoords, "vertices"));
1406: PetscCall(VecScale(newcoords, lengthScale));
1407: PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz"));
1408: PetscCall(PetscViewerHDF5WriteGroup(viewer, NULL));
1409: PetscCall(PetscViewerHDF5PopGroup(viewer));
1410: PetscCall(PetscViewerHDF5PushGroup(viewer, "/viz/geometry"));
1411: PetscCall(VecView(newcoords, viewer));
1412: PetscCall(PetscViewerHDF5PopGroup(viewer));
1413: PetscCall(VecDestroy(&newcoords));
1414: PetscFunctionReturn(PETSC_SUCCESS);
1415: }
1417: PetscErrorCode DMPlexLabelsView_HDF5_Internal(DM dm, IS globalPointNumbers, PetscViewer viewer)
1418: {
1419: const char *topologydm_name;
1420: const PetscInt *gpoint;
1421: PetscInt numLabels;
1422: PetscBool omitCelltypes = PETSC_FALSE;
1423: DMPlexStorageVersion version;
1424: char group[PETSC_MAX_PATH_LEN];
1426: PetscFunctionBegin;
1427: PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_omit_celltypes", &omitCelltypes, NULL));
1428: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionWriting(viewer, &version));
1429: PetscCall(ISGetIndices(globalPointNumbers, &gpoint));
1430: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1431: if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1432: PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s/labels", topologydm_name));
1433: } else {
1434: PetscCall(PetscStrncpy(group, "/labels", sizeof(group)));
1435: }
1436: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1437: PetscCall(DMGetNumLabels(dm, &numLabels));
1438: for (PetscInt l = 0; l < numLabels; ++l) {
1439: DMLabel label;
1440: const char *name;
1441: IS valueIS, pvalueIS, globalValueIS;
1442: const PetscInt *values;
1443: PetscInt numValues, v;
1444: PetscBool isDepth, isCelltype, output;
1446: PetscCall(DMGetLabelByNum(dm, l, &label));
1447: PetscCall(PetscObjectGetName((PetscObject)label, &name));
1448: PetscCall(DMGetLabelOutput(dm, name, &output));
1449: PetscCall(PetscStrncmp(name, "depth", 10, &isDepth));
1450: PetscCall(PetscStrncmp(name, "celltype", 10, &isCelltype));
1451: // TODO Should only filter out celltype if it can be calculated
1452: if (isDepth || (isCelltype && omitCelltypes) || !output) continue;
1453: PetscCall(PetscViewerHDF5PushGroup(viewer, name));
1454: PetscCall(DMLabelGetValueIS(label, &valueIS));
1455: /* Must copy to a new IS on the global comm */
1456: PetscCall(ISGetLocalSize(valueIS, &numValues));
1457: PetscCall(ISGetIndices(valueIS, &values));
1458: PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), numValues, values, PETSC_COPY_VALUES, &pvalueIS));
1459: PetscCall(ISRestoreIndices(valueIS, &values));
1460: PetscCall(ISAllGather(pvalueIS, &globalValueIS));
1461: PetscCall(ISDestroy(&pvalueIS));
1462: PetscCall(ISSortRemoveDups(globalValueIS));
1463: PetscCall(ISGetLocalSize(globalValueIS, &numValues));
1464: PetscCall(ISGetIndices(globalValueIS, &values));
1465: for (v = 0; v < numValues; ++v) {
1466: IS stratumIS, globalStratumIS;
1467: const PetscInt *spoints = NULL;
1468: PetscInt *gspoints, n = 0, gn, p;
1469: const char *iname = "indices";
1470: char group[PETSC_MAX_PATH_LEN];
1472: PetscCall(PetscSNPrintf(group, sizeof(group), "%" PetscInt_FMT, values[v]));
1473: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1474: PetscCall(DMLabelGetStratumIS(label, values[v], &stratumIS));
1476: if (stratumIS) PetscCall(ISGetLocalSize(stratumIS, &n));
1477: if (stratumIS) PetscCall(ISGetIndices(stratumIS, &spoints));
1478: for (gn = 0, p = 0; p < n; ++p)
1479: if (gpoint[spoints[p]] >= 0) ++gn;
1480: PetscCall(PetscMalloc1(gn, &gspoints));
1481: for (gn = 0, p = 0; p < n; ++p)
1482: if (gpoint[spoints[p]] >= 0) gspoints[gn++] = gpoint[spoints[p]];
1483: if (stratumIS) PetscCall(ISRestoreIndices(stratumIS, &spoints));
1484: PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), gn, gspoints, PETSC_OWN_POINTER, &globalStratumIS));
1485: PetscCall(PetscObjectSetName((PetscObject)globalStratumIS, iname));
1487: PetscCall(ISView(globalStratumIS, viewer));
1488: PetscCall(ISDestroy(&globalStratumIS));
1489: PetscCall(ISDestroy(&stratumIS));
1490: PetscCall(PetscViewerHDF5PopGroup(viewer));
1491: }
1492: PetscCall(ISRestoreIndices(globalValueIS, &values));
1493: PetscCall(ISDestroy(&globalValueIS));
1494: PetscCall(ISDestroy(&valueIS));
1495: PetscCall(PetscViewerHDF5PopGroup(viewer));
1496: }
1497: PetscCall(ISRestoreIndices(globalPointNumbers, &gpoint));
1498: PetscCall(PetscViewerHDF5PopGroup(viewer));
1499: PetscFunctionReturn(PETSC_SUCCESS);
1500: }
1502: /* We only write cells and vertices. Does this screw up parallel reading? */
1503: PetscErrorCode DMPlexView_HDF5_Internal(DM dm, PetscViewer viewer)
1504: {
1505: IS globalPointNumbers;
1506: PetscViewerFormat format;
1507: PetscBool viz_geom = PETSC_FALSE, xdmf_topo = PETSC_FALSE, petsc_topo = PETSC_FALSE;
1509: PetscFunctionBegin;
1510: PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbers));
1511: PetscCall(DMPlexCoordinatesView_HDF5_Internal(dm, viewer));
1513: PetscCall(PetscViewerGetFormat(viewer, &format));
1514: switch (format) {
1515: case PETSC_VIEWER_HDF5_VIZ:
1516: viz_geom = PETSC_TRUE;
1517: xdmf_topo = PETSC_TRUE;
1518: break;
1519: case PETSC_VIEWER_HDF5_XDMF:
1520: xdmf_topo = PETSC_TRUE;
1521: break;
1522: case PETSC_VIEWER_HDF5_PETSC:
1523: petsc_topo = PETSC_TRUE;
1524: break;
1525: case PETSC_VIEWER_DEFAULT:
1526: case PETSC_VIEWER_NATIVE:
1527: viz_geom = PETSC_TRUE;
1528: xdmf_topo = PETSC_TRUE;
1529: petsc_topo = PETSC_TRUE;
1530: break;
1531: default:
1532: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1533: }
1535: if (viz_geom) PetscCall(DMPlexCoordinatesView_HDF5_XDMF_Private(dm, viewer));
1536: if (xdmf_topo) PetscCall(DMPlexTopologyView_HDF5_XDMF_Private(dm, globalPointNumbers, viewer));
1537: if (petsc_topo) {
1538: PetscBool viewLabels = PETSC_TRUE;
1540: PetscCall(DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbers, viewer));
1541: PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_view_labels", &viewLabels, NULL));
1542: if (viewLabels) PetscCall(DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbers, viewer));
1543: }
1545: PetscCall(ISDestroy(&globalPointNumbers));
1546: PetscFunctionReturn(PETSC_SUCCESS);
1547: }
1549: PetscErrorCode DMPlexSectionView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm)
1550: {
1551: MPI_Comm comm;
1552: const char *topologydm_name;
1553: const char *sectiondm_name;
1554: PetscSection gsection;
1556: PetscFunctionBegin;
1557: PetscCall(PetscObjectGetComm((PetscObject)sectiondm, &comm));
1558: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1559: PetscCall(PetscObjectGetName((PetscObject)sectiondm, §iondm_name));
1560: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1561: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1562: PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1563: PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1564: PetscCall(DMGetGlobalSection(sectiondm, &gsection));
1565: /* Save raw section */
1566: PetscCall(PetscSectionView(gsection, viewer));
1567: /* Save plex wrapper */
1568: {
1569: PetscInt pStart, pEnd, p, n;
1570: IS globalPointNumbers;
1571: const PetscInt *gpoints;
1572: IS orderIS;
1573: PetscInt *order;
1575: PetscCall(PetscSectionGetChart(gsection, &pStart, &pEnd));
1576: PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbers));
1577: PetscCall(ISGetIndices(globalPointNumbers, &gpoints));
1578: for (p = pStart, n = 0; p < pEnd; ++p)
1579: if (gpoints[p] >= 0) n++;
1580: /* "order" is an array of global point numbers.
1581: When loading, it is used with topology/order array
1582: to match section points with plex topology points. */
1583: PetscCall(PetscMalloc1(n, &order));
1584: for (p = pStart, n = 0; p < pEnd; ++p)
1585: if (gpoints[p] >= 0) order[n++] = gpoints[p];
1586: PetscCall(ISRestoreIndices(globalPointNumbers, &gpoints));
1587: PetscCall(ISDestroy(&globalPointNumbers));
1588: PetscCall(ISCreateGeneral(comm, n, order, PETSC_OWN_POINTER, &orderIS));
1589: PetscCall(PetscObjectSetName((PetscObject)orderIS, "order"));
1590: PetscCall(ISView(orderIS, viewer));
1591: PetscCall(ISDestroy(&orderIS));
1592: }
1593: PetscCall(PetscViewerHDF5PopGroup(viewer));
1594: PetscCall(PetscViewerHDF5PopGroup(viewer));
1595: PetscCall(PetscViewerHDF5PopGroup(viewer));
1596: PetscCall(PetscViewerHDF5PopGroup(viewer));
1597: PetscFunctionReturn(PETSC_SUCCESS);
1598: }
1600: PetscErrorCode DMPlexGlobalVectorView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1601: {
1602: const char *topologydm_name;
1603: const char *sectiondm_name;
1604: const char *vec_name;
1605: PetscInt bs;
1607: PetscFunctionBegin;
1608: /* Check consistency */
1609: {
1610: PetscSF pointsf, pointsf1;
1612: PetscCall(DMGetPointSF(dm, &pointsf));
1613: PetscCall(DMGetPointSF(sectiondm, &pointsf1));
1614: PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
1615: }
1616: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1617: PetscCall(PetscObjectGetName((PetscObject)sectiondm, §iondm_name));
1618: PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
1619: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1620: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1621: PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1622: PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1623: PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
1624: PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
1625: PetscCall(VecGetBlockSize(vec, &bs));
1626: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "blockSize", PETSC_INT, (void *)&bs));
1627: PetscCall(VecSetBlockSize(vec, 1));
1628: /* VecView(vec, viewer) would call (*vec->opt->view)(vec, viewer), but, */
1629: /* if vec was created with DMGet{Global, Local}Vector(), vec->opt->view */
1630: /* is set to VecView_Plex, which would save vec in a predefined location. */
1631: /* To save vec in where we want, we create a new Vec (temp) with */
1632: /* VecCreate(), wrap the vec data in temp, and call VecView(temp, viewer). */
1633: {
1634: Vec temp;
1635: const PetscScalar *array;
1636: PetscLayout map;
1638: PetscCall(VecCreate(PetscObjectComm((PetscObject)vec), &temp));
1639: PetscCall(PetscObjectSetName((PetscObject)temp, vec_name));
1640: PetscCall(VecGetLayout(vec, &map));
1641: PetscCall(VecSetLayout(temp, map));
1642: PetscCall(VecSetUp(temp));
1643: PetscCall(VecGetArrayRead(vec, &array));
1644: PetscCall(VecPlaceArray(temp, array));
1645: PetscCall(VecView(temp, viewer));
1646: PetscCall(VecResetArray(temp));
1647: PetscCall(VecRestoreArrayRead(vec, &array));
1648: PetscCall(VecDestroy(&temp));
1649: }
1650: PetscCall(VecSetBlockSize(vec, bs));
1651: PetscCall(PetscViewerHDF5PopGroup(viewer));
1652: PetscCall(PetscViewerHDF5PopGroup(viewer));
1653: PetscCall(PetscViewerHDF5PopGroup(viewer));
1654: PetscCall(PetscViewerHDF5PopGroup(viewer));
1655: PetscCall(PetscViewerHDF5PopGroup(viewer));
1656: PetscCall(PetscViewerHDF5PopGroup(viewer));
1657: PetscFunctionReturn(PETSC_SUCCESS);
1658: }
1660: PetscErrorCode DMPlexLocalVectorView_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
1661: {
1662: MPI_Comm comm;
1663: const char *topologydm_name;
1664: const char *sectiondm_name;
1665: const char *vec_name;
1666: PetscSection section;
1667: PetscBool includesConstraints;
1668: Vec gvec;
1669: PetscInt m, bs;
1671: PetscFunctionBegin;
1672: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1673: /* Check consistency */
1674: {
1675: PetscSF pointsf, pointsf1;
1677: PetscCall(DMGetPointSF(dm, &pointsf));
1678: PetscCall(DMGetPointSF(sectiondm, &pointsf1));
1679: PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
1680: }
1681: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1682: PetscCall(PetscObjectGetName((PetscObject)sectiondm, §iondm_name));
1683: PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
1684: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
1685: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
1686: PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
1687: PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
1688: PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
1689: PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
1690: PetscCall(VecGetBlockSize(vec, &bs));
1691: PetscCall(PetscViewerHDF5WriteAttribute(viewer, NULL, "blockSize", PETSC_INT, (void *)&bs));
1692: PetscCall(VecCreate(comm, &gvec));
1693: PetscCall(PetscObjectSetName((PetscObject)gvec, vec_name));
1694: PetscCall(DMGetGlobalSection(sectiondm, §ion));
1695: PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
1696: if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
1697: else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
1698: PetscCall(VecSetSizes(gvec, m, PETSC_DECIDE));
1699: PetscCall(VecSetUp(gvec));
1700: PetscCall(DMLocalToGlobalBegin(sectiondm, vec, INSERT_VALUES, gvec));
1701: PetscCall(DMLocalToGlobalEnd(sectiondm, vec, INSERT_VALUES, gvec));
1702: PetscCall(VecView(gvec, viewer));
1703: PetscCall(VecDestroy(&gvec));
1704: PetscCall(PetscViewerHDF5PopGroup(viewer));
1705: PetscCall(PetscViewerHDF5PopGroup(viewer));
1706: PetscCall(PetscViewerHDF5PopGroup(viewer));
1707: PetscCall(PetscViewerHDF5PopGroup(viewer));
1708: PetscCall(PetscViewerHDF5PopGroup(viewer));
1709: PetscCall(PetscViewerHDF5PopGroup(viewer));
1710: PetscFunctionReturn(PETSC_SUCCESS);
1711: }
1713: struct _n_LoadLabelsCtx {
1714: MPI_Comm comm;
1715: PetscMPIInt rank;
1716: DM dm;
1717: PetscViewer viewer;
1718: DMLabel label;
1719: PetscSF sfXC;
1720: PetscLayout layoutX;
1721: };
1722: typedef struct _n_LoadLabelsCtx *LoadLabelsCtx;
1724: static PetscErrorCode LoadLabelsCtxCreate(DM dm, PetscViewer viewer, PetscSF sfXC, LoadLabelsCtx *ctx)
1725: {
1726: PetscFunctionBegin;
1727: PetscCall(PetscNew(ctx));
1728: PetscCall(PetscObjectReference((PetscObject)((*ctx)->dm = dm)));
1729: PetscCall(PetscObjectReference((PetscObject)((*ctx)->viewer = viewer)));
1730: PetscCall(PetscObjectGetComm((PetscObject)dm, &(*ctx)->comm));
1731: PetscCallMPI(MPI_Comm_rank((*ctx)->comm, &(*ctx)->rank));
1732: (*ctx)->sfXC = sfXC;
1733: if (sfXC) {
1734: PetscInt nX;
1736: PetscCall(PetscObjectReference((PetscObject)sfXC));
1737: PetscCall(PetscSFGetGraph(sfXC, &nX, NULL, NULL, NULL));
1738: PetscCall(PetscLayoutCreateFromSizes((*ctx)->comm, nX, PETSC_DECIDE, 1, &(*ctx)->layoutX));
1739: }
1740: PetscFunctionReturn(PETSC_SUCCESS);
1741: }
1743: static PetscErrorCode LoadLabelsCtxDestroy(LoadLabelsCtx *ctx)
1744: {
1745: PetscFunctionBegin;
1746: if (!*ctx) PetscFunctionReturn(PETSC_SUCCESS);
1747: PetscCall(DMDestroy(&(*ctx)->dm));
1748: PetscCall(PetscViewerDestroy(&(*ctx)->viewer));
1749: PetscCall(PetscSFDestroy(&(*ctx)->sfXC));
1750: PetscCall(PetscLayoutDestroy(&(*ctx)->layoutX));
1751: PetscCall(PetscFree(*ctx));
1752: PetscFunctionReturn(PETSC_SUCCESS);
1753: }
1755: /*
1756: A: on-disk points
1757: X: global points [0, NX)
1758: C: distributed plex points
1759: */
1760: static herr_t ReadLabelStratumHDF5_Distribute_Private(IS stratumIS, LoadLabelsCtx ctx, IS *newStratumIS)
1761: {
1762: MPI_Comm comm = ctx->comm;
1763: PetscSF sfXC = ctx->sfXC;
1764: PetscLayout layoutX = ctx->layoutX;
1765: PetscSF sfXA;
1766: const PetscInt *A_points;
1767: PetscInt nX, nC;
1768: PetscInt n;
1770: PetscFunctionBegin;
1771: PetscCall(PetscSFGetGraph(sfXC, &nX, &nC, NULL, NULL));
1772: PetscCall(ISGetLocalSize(stratumIS, &n));
1773: PetscCall(ISGetIndices(stratumIS, &A_points));
1774: PetscCall(PetscSFCreate(comm, &sfXA));
1775: PetscCall(PetscSFSetGraphLayout(sfXA, layoutX, n, NULL, PETSC_USE_POINTER, A_points));
1776: PetscCall(ISCreate(comm, newStratumIS));
1777: PetscCall(ISSetType(*newStratumIS, ISGENERAL));
1778: {
1779: PetscInt i;
1780: PetscBool *A_mask, *X_mask, *C_mask;
1782: PetscCall(PetscCalloc3(n, &A_mask, nX, &X_mask, nC, &C_mask));
1783: for (i = 0; i < n; i++) A_mask[i] = PETSC_TRUE;
1784: PetscCall(PetscSFReduceBegin(sfXA, MPIU_BOOL, A_mask, X_mask, MPI_REPLACE));
1785: PetscCall(PetscSFReduceEnd(sfXA, MPIU_BOOL, A_mask, X_mask, MPI_REPLACE));
1786: PetscCall(PetscSFBcastBegin(sfXC, MPIU_BOOL, X_mask, C_mask, MPI_LOR));
1787: PetscCall(PetscSFBcastEnd(sfXC, MPIU_BOOL, X_mask, C_mask, MPI_LOR));
1788: PetscCall(ISGeneralSetIndicesFromMask(*newStratumIS, 0, nC, C_mask));
1789: PetscCall(PetscFree3(A_mask, X_mask, C_mask));
1790: }
1791: PetscCall(PetscSFDestroy(&sfXA));
1792: PetscCall(ISRestoreIndices(stratumIS, &A_points));
1793: PetscFunctionReturn(PETSC_SUCCESS);
1794: }
1796: static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *vname, const H5L_info_t *info, void *op_data)
1797: {
1798: LoadLabelsCtx ctx = (LoadLabelsCtx)op_data;
1799: PetscViewer viewer = ctx->viewer;
1800: DMLabel label = ctx->label;
1801: MPI_Comm comm = ctx->comm;
1802: IS stratumIS;
1803: const PetscInt *ind;
1804: PetscInt value, N, i;
1806: PetscCall(PetscOptionsStringToInt(vname, &value));
1807: PetscCall(ISCreate(comm, &stratumIS));
1808: PetscCall(PetscObjectSetName((PetscObject)stratumIS, "indices"));
1809: PetscCall(PetscViewerHDF5PushGroup(viewer, vname)); /* labels/<lname>/<vname> */
1811: if (!ctx->sfXC) {
1812: /* Force serial load */
1813: PetscCall(PetscViewerHDF5ReadSizes(viewer, "indices", NULL, &N));
1814: PetscCall(PetscLayoutSetLocalSize(stratumIS->map, !ctx->rank ? N : 0));
1815: PetscCall(PetscLayoutSetSize(stratumIS->map, N));
1816: }
1817: PetscCall(ISLoad(stratumIS, viewer));
1819: if (ctx->sfXC) {
1820: IS newStratumIS;
1822: PetscCallHDF5(ReadLabelStratumHDF5_Distribute_Private, (stratumIS, ctx, &newStratumIS));
1823: PetscCall(ISDestroy(&stratumIS));
1824: stratumIS = newStratumIS;
1825: }
1827: PetscCall(PetscViewerHDF5PopGroup(viewer));
1828: PetscCall(ISGetLocalSize(stratumIS, &N));
1829: PetscCall(ISGetIndices(stratumIS, &ind));
1830: for (i = 0; i < N; ++i) PetscCall(DMLabelSetValue(label, ind[i], value));
1831: PetscCall(ISRestoreIndices(stratumIS, &ind));
1832: PetscCall(ISDestroy(&stratumIS));
1833: return 0;
1834: }
1836: /* TODO: Fix this code, it is returning PETSc error codes when it should be translating them to herr_t codes */
1837: static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *lname, const H5L_info_t *info, void *op_data)
1838: {
1839: LoadLabelsCtx ctx = (LoadLabelsCtx)op_data;
1840: DM dm = ctx->dm;
1841: hsize_t idx = 0;
1842: PetscErrorCode ierr;
1843: PetscBool flg;
1844: herr_t err;
1846: PetscCall(DMHasLabel(dm, lname, &flg));
1847: if (flg) PetscCall(DMRemoveLabel(dm, lname, NULL));
1848: ierr = DMCreateLabel(dm, lname);
1849: if (ierr) return (herr_t)ierr;
1850: ierr = DMGetLabel(dm, lname, &ctx->label);
1851: if (ierr) return (herr_t)ierr;
1852: ierr = PetscViewerHDF5PushGroup(ctx->viewer, lname);
1853: if (ierr) return (herr_t)ierr;
1854: /* Iterate over the label's strata */
1855: PetscCallHDF5Return(err, H5Literate_by_name, (g_id, lname, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0));
1856: ierr = PetscViewerHDF5PopGroup(ctx->viewer);
1857: if (ierr) return (herr_t)ierr;
1858: return err;
1859: }
1861: PetscErrorCode DMPlexLabelsLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF sfXC)
1862: {
1863: const char *topologydm_name;
1864: LoadLabelsCtx ctx;
1865: hsize_t idx = 0;
1866: char group[PETSC_MAX_PATH_LEN];
1867: DMPlexStorageVersion version;
1868: PetscBool distributed, hasGroup;
1870: PetscFunctionBegin;
1871: PetscCall(DMPlexIsDistributed(dm, &distributed));
1872: if (distributed) PetscCheck(sfXC, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_NULL, "PetscSF must be given for parallel load");
1873: PetscCall(LoadLabelsCtxCreate(dm, viewer, sfXC, &ctx));
1874: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
1875: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
1876: if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
1877: PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s/labels", topologydm_name));
1878: } else {
1879: PetscCall(PetscStrncpy(group, "labels", sizeof(group)));
1880: }
1881: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
1882: PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &hasGroup));
1883: if (hasGroup) {
1884: hid_t fileId, groupId;
1886: PetscCall(PetscViewerHDF5OpenGroup(viewer, NULL, &fileId, &groupId));
1887: /* Iterate over labels */
1888: PetscCallHDF5(H5Literate, (groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, ctx));
1889: PetscCallHDF5(H5Gclose, (groupId));
1890: }
1891: PetscCall(PetscViewerHDF5PopGroup(viewer));
1892: PetscCall(LoadLabelsCtxDestroy(&ctx));
1893: PetscFunctionReturn(PETSC_SUCCESS);
1894: }
1896: static PetscErrorCode DMPlexDistributionLoad_HDF5_Private(DM dm, PetscViewer viewer, PetscSF sf, PetscSF *distsf, DM *distdm)
1897: {
1898: MPI_Comm comm;
1899: PetscMPIInt size, rank;
1900: PetscInt dist_size;
1901: const char *distribution_name;
1902: PetscInt p, lsize;
1903: IS chartSizesIS, ownersIS, gpointsIS;
1904: const PetscInt *chartSize, *owners, *gpoints;
1905: PetscLayout layout;
1906: PetscBool has;
1908: PetscFunctionBegin;
1909: *distsf = NULL;
1910: *distdm = NULL;
1911: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1912: PetscCallMPI(MPI_Comm_size(comm, &size));
1913: PetscCallMPI(MPI_Comm_rank(comm, &rank));
1914: PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
1915: if (!distribution_name) PetscFunctionReturn(PETSC_SUCCESS);
1916: PetscCall(PetscLogEventBegin(DMPLEX_DistributionLoad, viewer, 0, 0, 0));
1917: PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &has));
1918: if (!has) {
1919: char *full_group;
1921: PetscCall(PetscViewerHDF5GetGroup(viewer, NULL, &full_group));
1922: PetscCheck(has, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Distribution %s cannot be found: HDF5 group %s not found in file", distribution_name, full_group);
1923: }
1924: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "comm_size", PETSC_INT, NULL, (void *)&dist_size));
1925: PetscCheck(dist_size == (PetscInt)size, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Mismatching comm sizes: comm size of this session (%d) != comm size used for %s (%" PetscInt_FMT ")", size, distribution_name, dist_size);
1926: PetscCall(ISCreate(comm, &chartSizesIS));
1927: PetscCall(PetscObjectSetName((PetscObject)chartSizesIS, "chart_sizes"));
1928: PetscCall(ISCreate(comm, &ownersIS));
1929: PetscCall(PetscObjectSetName((PetscObject)ownersIS, "owners"));
1930: PetscCall(ISCreate(comm, &gpointsIS));
1931: PetscCall(PetscObjectSetName((PetscObject)gpointsIS, "global_point_numbers"));
1932: PetscCall(PetscLayoutSetLocalSize(chartSizesIS->map, 1));
1933: PetscCall(ISLoad(chartSizesIS, viewer));
1934: PetscCall(ISGetIndices(chartSizesIS, &chartSize));
1935: PetscCall(PetscLayoutSetLocalSize(ownersIS->map, *chartSize));
1936: PetscCall(PetscLayoutSetLocalSize(gpointsIS->map, *chartSize));
1937: PetscCall(ISLoad(ownersIS, viewer));
1938: PetscCall(ISLoad(gpointsIS, viewer));
1939: PetscCall(ISGetIndices(ownersIS, &owners));
1940: PetscCall(ISGetIndices(gpointsIS, &gpoints));
1941: PetscCall(PetscSFCreate(comm, distsf));
1942: PetscCall(PetscSFSetFromOptions(*distsf));
1943: PetscCall(PetscLayoutCreate(comm, &layout));
1944: PetscCall(PetscSFGetGraph(sf, &lsize, NULL, NULL, NULL));
1945: PetscCall(PetscLayoutSetLocalSize(layout, lsize));
1946: PetscCall(PetscLayoutSetBlockSize(layout, 1));
1947: PetscCall(PetscLayoutSetUp(layout));
1948: PetscCall(PetscSFSetGraphLayout(*distsf, layout, *chartSize, NULL, PETSC_OWN_POINTER, gpoints));
1949: PetscCall(PetscLayoutDestroy(&layout));
1950: /* Migrate DM */
1951: {
1952: PetscInt pStart, pEnd;
1953: PetscSFNode *buffer0, *buffer1, *buffer2;
1955: PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1956: PetscCall(PetscMalloc2(pEnd - pStart, &buffer0, lsize, &buffer1));
1957: PetscCall(PetscMalloc1(*chartSize, &buffer2));
1958: {
1959: PetscSF workPointSF;
1960: PetscInt workNroots, workNleaves;
1961: const PetscInt *workIlocal;
1962: const PetscSFNode *workIremote;
1964: for (p = pStart; p < pEnd; ++p) {
1965: buffer0[p - pStart].rank = rank;
1966: buffer0[p - pStart].index = p - pStart;
1967: }
1968: PetscCall(DMGetPointSF(dm, &workPointSF));
1969: PetscCall(PetscSFGetGraph(workPointSF, &workNroots, &workNleaves, &workIlocal, &workIremote));
1970: for (p = 0; p < workNleaves; ++p) {
1971: PetscInt workIlocalp = (workIlocal ? workIlocal[p] : p);
1973: buffer0[workIlocalp].rank = -1;
1974: }
1975: }
1976: for (p = 0; p < lsize; ++p) buffer1[p].rank = -1;
1977: for (p = 0; p < *chartSize; ++p) buffer2[p].rank = -1;
1978: PetscCall(PetscSFReduceBegin(sf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
1979: PetscCall(PetscSFReduceEnd(sf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
1980: PetscCall(PetscSFBcastBegin(*distsf, MPIU_SF_NODE, buffer1, buffer2, MPI_REPLACE));
1981: PetscCall(PetscSFBcastEnd(*distsf, MPIU_SF_NODE, buffer1, buffer2, MPI_REPLACE));
1982: if (PetscDefined(USE_DEBUG)) {
1983: for (p = 0; p < *chartSize; ++p) {
1984: PetscCheck(buffer2[p].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Found negative root rank %" PetscInt_FMT " at local point %" PetscInt_FMT " on rank %d when making migrationSF", buffer2[p].rank, p, rank);
1985: }
1986: }
1987: PetscCall(PetscFree2(buffer0, buffer1));
1988: PetscCall(DMCreate(comm, distdm));
1989: PetscCall(DMSetType(*distdm, DMPLEX));
1990: PetscCall(PetscObjectSetName((PetscObject)*distdm, ((PetscObject)dm)->name));
1991: PetscCall(DMPlexDistributionSetName(*distdm, distribution_name));
1992: {
1993: PetscSF migrationSF;
1995: PetscCall(PetscSFCreate(comm, &migrationSF));
1996: PetscCall(PetscSFSetFromOptions(migrationSF));
1997: PetscCall(PetscSFSetGraph(migrationSF, pEnd - pStart, *chartSize, NULL, PETSC_OWN_POINTER, buffer2, PETSC_OWN_POINTER));
1998: PetscCall(PetscSFSetUp(migrationSF));
1999: PetscCall(DMPlexMigrate(dm, migrationSF, *distdm));
2000: PetscCall(PetscSFDestroy(&migrationSF));
2001: }
2002: }
2003: /* Set pointSF */
2004: {
2005: PetscSF pointSF;
2006: PetscInt *ilocal, nleaves, q;
2007: PetscSFNode *iremote, *buffer0, *buffer1;
2009: PetscCall(PetscMalloc2(*chartSize, &buffer0, lsize, &buffer1));
2010: for (p = 0, nleaves = 0; p < *chartSize; ++p) {
2011: if (owners[p] == rank) {
2012: buffer0[p].rank = rank;
2013: } else {
2014: buffer0[p].rank = -1;
2015: nleaves++;
2016: }
2017: buffer0[p].index = p;
2018: }
2019: for (p = 0; p < lsize; ++p) buffer1[p].rank = -1;
2020: PetscCall(PetscSFReduceBegin(*distsf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
2021: PetscCall(PetscSFReduceEnd(*distsf, MPIU_SF_NODE, buffer0, buffer1, MPI_MAXLOC));
2022: for (p = 0; p < *chartSize; ++p) buffer0[p].rank = -1;
2023: PetscCall(PetscSFBcastBegin(*distsf, MPIU_SF_NODE, buffer1, buffer0, MPI_REPLACE));
2024: PetscCall(PetscSFBcastEnd(*distsf, MPIU_SF_NODE, buffer1, buffer0, MPI_REPLACE));
2025: if (PetscDefined(USE_DEBUG)) {
2026: for (p = 0; p < *chartSize; ++p) PetscCheck(buffer0[p].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Found negative root rank %" PetscInt_FMT " at local point %" PetscInt_FMT " on rank %d when making pointSF", buffer0[p].rank, p, rank);
2027: }
2028: PetscCall(PetscMalloc1(nleaves, &ilocal));
2029: PetscCall(PetscMalloc1(nleaves, &iremote));
2030: for (p = 0, q = 0; p < *chartSize; ++p) {
2031: if (buffer0[p].rank != rank) {
2032: ilocal[q] = p;
2033: iremote[q].rank = buffer0[p].rank;
2034: iremote[q].index = buffer0[p].index;
2035: q++;
2036: }
2037: }
2038: PetscCheck(q == nleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching leaf sizes: %" PetscInt_FMT " != %" PetscInt_FMT, q, nleaves);
2039: PetscCall(PetscFree2(buffer0, buffer1));
2040: PetscCall(PetscSFCreate(comm, &pointSF));
2041: PetscCall(PetscSFSetFromOptions(pointSF));
2042: PetscCall(PetscSFSetGraph(pointSF, *chartSize, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
2043: PetscCall(DMSetPointSF(*distdm, pointSF));
2044: {
2045: DM cdm;
2047: PetscCall(DMGetCoordinateDM(*distdm, &cdm));
2048: PetscCall(DMSetPointSF(cdm, pointSF));
2049: }
2050: PetscCall(PetscSFDestroy(&pointSF));
2051: }
2052: PetscCall(ISRestoreIndices(chartSizesIS, &chartSize));
2053: PetscCall(ISRestoreIndices(ownersIS, &owners));
2054: PetscCall(ISRestoreIndices(gpointsIS, &gpoints));
2055: PetscCall(ISDestroy(&chartSizesIS));
2056: PetscCall(ISDestroy(&ownersIS));
2057: PetscCall(ISDestroy(&gpointsIS));
2058: /* Record that overlap has been manually created. */
2059: /* This is to pass `DMPlexCheckPointSF()`, which checks that */
2060: /* pointSF does not contain cells in the leaves if overlap = 0. */
2061: PetscCall(DMPlexSetOverlap_Plex(*distdm, NULL, DMPLEX_OVERLAP_MANUAL));
2062: PetscCall(DMPlexDistributeSetDefault(*distdm, PETSC_FALSE));
2063: PetscCall(DMPlexReorderSetDefault(*distdm, DM_REORDER_DEFAULT_FALSE));
2064: PetscCall(PetscLogEventEnd(DMPLEX_DistributionLoad, viewer, 0, 0, 0));
2065: PetscFunctionReturn(PETSC_SUCCESS);
2066: }
2068: // Serial load of topology
2069: static PetscErrorCode DMPlexTopologyLoad_HDF5_Legacy_Private(DM dm, PetscViewer viewer, PetscSF *sf)
2070: {
2071: MPI_Comm comm;
2072: const char *pointsName, *coneSizesName, *conesName, *orientationsName;
2073: IS pointsIS, coneSizesIS, conesIS, orientationsIS;
2074: const PetscInt *points, *coneSizes, *cones, *orientations;
2075: PetscInt *cone, *ornt;
2076: PetscInt dim, N, Np, pEnd, p, q, maxConeSize = 0, c;
2077: PetscMPIInt size, rank;
2079: PetscFunctionBegin;
2080: pointsName = "order";
2081: coneSizesName = "cones";
2082: conesName = "cells";
2083: orientationsName = "orientation";
2084: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2085: PetscCallMPI(MPI_Comm_size(comm, &size));
2086: PetscCallMPI(MPI_Comm_rank(comm, &rank));
2087: PetscCall(ISCreate(comm, &pointsIS));
2088: PetscCall(PetscObjectSetName((PetscObject)pointsIS, pointsName));
2089: PetscCall(ISCreate(comm, &coneSizesIS));
2090: PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
2091: PetscCall(ISCreate(comm, &conesIS));
2092: PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
2093: PetscCall(ISCreate(comm, &orientationsIS));
2094: PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
2095: PetscCall(PetscViewerHDF5ReadObjectAttribute(viewer, (PetscObject)conesIS, "cell_dim", PETSC_INT, NULL, &dim));
2096: PetscCall(DMSetDimension(dm, dim));
2097: {
2098: /* Force serial load */
2099: PetscCall(PetscInfo(dm, "Loading DM %s in serial\n", dm->hdr.name));
2100: PetscCall(PetscViewerHDF5ReadSizes(viewer, pointsName, NULL, &Np));
2101: PetscCall(PetscLayoutSetLocalSize(pointsIS->map, rank == 0 ? Np : 0));
2102: PetscCall(PetscLayoutSetSize(pointsIS->map, Np));
2103: pEnd = rank == 0 ? Np : 0;
2104: PetscCall(PetscViewerHDF5ReadSizes(viewer, coneSizesName, NULL, &Np));
2105: PetscCall(PetscLayoutSetLocalSize(coneSizesIS->map, rank == 0 ? Np : 0));
2106: PetscCall(PetscLayoutSetSize(coneSizesIS->map, Np));
2107: PetscCall(PetscViewerHDF5ReadSizes(viewer, conesName, NULL, &N));
2108: PetscCall(PetscLayoutSetLocalSize(conesIS->map, rank == 0 ? N : 0));
2109: PetscCall(PetscLayoutSetSize(conesIS->map, N));
2110: PetscCall(PetscViewerHDF5ReadSizes(viewer, orientationsName, NULL, &N));
2111: PetscCall(PetscLayoutSetLocalSize(orientationsIS->map, rank == 0 ? N : 0));
2112: PetscCall(PetscLayoutSetSize(orientationsIS->map, N));
2113: }
2114: PetscCall(ISLoad(pointsIS, viewer));
2115: PetscCall(ISLoad(coneSizesIS, viewer));
2116: PetscCall(ISLoad(conesIS, viewer));
2117: PetscCall(ISLoad(orientationsIS, viewer));
2118: /* Create Plex */
2119: PetscCall(DMPlexSetChart(dm, 0, pEnd));
2120: PetscCall(ISGetIndices(pointsIS, &points));
2121: PetscCall(ISGetIndices(coneSizesIS, &coneSizes));
2122: for (p = 0; p < pEnd; ++p) {
2123: PetscCall(DMPlexSetConeSize(dm, points[p], coneSizes[p]));
2124: maxConeSize = PetscMax(maxConeSize, coneSizes[p]);
2125: }
2126: PetscCall(DMSetUp(dm));
2127: PetscCall(ISGetIndices(conesIS, &cones));
2128: PetscCall(ISGetIndices(orientationsIS, &orientations));
2129: PetscCall(PetscMalloc2(maxConeSize, &cone, maxConeSize, &ornt));
2130: for (p = 0, q = 0; p < pEnd; ++p) {
2131: for (c = 0; c < coneSizes[p]; ++c, ++q) {
2132: cone[c] = cones[q];
2133: ornt[c] = orientations[q];
2134: }
2135: PetscCall(DMPlexSetCone(dm, points[p], cone));
2136: PetscCall(DMPlexSetConeOrientation(dm, points[p], ornt));
2137: }
2138: PetscCall(PetscFree2(cone, ornt));
2139: /* Create global section migration SF */
2140: if (sf) {
2141: PetscLayout layout;
2142: PetscInt *globalIndices;
2144: PetscCall(PetscMalloc1(pEnd, &globalIndices));
2145: /* plex point == globalPointNumber in this case */
2146: for (p = 0; p < pEnd; ++p) globalIndices[p] = p;
2147: PetscCall(PetscLayoutCreate(comm, &layout));
2148: PetscCall(PetscLayoutSetSize(layout, Np));
2149: PetscCall(PetscLayoutSetBlockSize(layout, 1));
2150: PetscCall(PetscLayoutSetUp(layout));
2151: PetscCall(PetscSFCreate(comm, sf));
2152: PetscCall(PetscSFSetFromOptions(*sf));
2153: PetscCall(PetscSFSetGraphLayout(*sf, layout, pEnd, NULL, PETSC_OWN_POINTER, globalIndices));
2154: PetscCall(PetscLayoutDestroy(&layout));
2155: PetscCall(PetscFree(globalIndices));
2156: }
2157: /* Clean-up */
2158: PetscCall(ISRestoreIndices(pointsIS, &points));
2159: PetscCall(ISRestoreIndices(coneSizesIS, &coneSizes));
2160: PetscCall(ISRestoreIndices(conesIS, &cones));
2161: PetscCall(ISRestoreIndices(orientationsIS, &orientations));
2162: PetscCall(ISDestroy(&pointsIS));
2163: PetscCall(ISDestroy(&coneSizesIS));
2164: PetscCall(ISDestroy(&conesIS));
2165: PetscCall(ISDestroy(&orientationsIS));
2166: /* Fill in the rest of the topology structure */
2167: PetscCall(DMPlexSymmetrize(dm));
2168: PetscCall(DMPlexStratify(dm));
2169: PetscFunctionReturn(PETSC_SUCCESS);
2170: }
2172: /* Representation of two DMPlex strata in 0-based global numbering */
2173: struct _n_PlexLayer {
2174: PetscInt d;
2175: IS conesIS, orientationsIS;
2176: PetscSection coneSizesSection;
2177: PetscLayout vertexLayout;
2178: PetscSF overlapSF, l2gSF; //TODO maybe confusing names (in DMPlex in general)
2179: PetscInt offset, conesOffset, leafOffset;
2180: };
2181: typedef struct _n_PlexLayer *PlexLayer;
2183: static PetscErrorCode PlexLayerDestroy(PlexLayer *layer)
2184: {
2185: PetscFunctionBegin;
2186: if (!*layer) PetscFunctionReturn(PETSC_SUCCESS);
2187: PetscCall(PetscSectionDestroy(&(*layer)->coneSizesSection));
2188: PetscCall(ISDestroy(&(*layer)->conesIS));
2189: PetscCall(ISDestroy(&(*layer)->orientationsIS));
2190: PetscCall(PetscSFDestroy(&(*layer)->overlapSF));
2191: PetscCall(PetscSFDestroy(&(*layer)->l2gSF));
2192: PetscCall(PetscLayoutDestroy(&(*layer)->vertexLayout));
2193: PetscCall(PetscFree(*layer));
2194: PetscFunctionReturn(PETSC_SUCCESS);
2195: }
2197: static PetscErrorCode PlexLayerCreate_Private(PlexLayer *layer)
2198: {
2199: PetscFunctionBegin;
2200: PetscCall(PetscNew(layer));
2201: (*layer)->d = -1;
2202: (*layer)->offset = -1;
2203: (*layer)->conesOffset = -1;
2204: (*layer)->leafOffset = -1;
2205: PetscFunctionReturn(PETSC_SUCCESS);
2206: }
2208: // Parallel load of a depth stratum
2209: static PetscErrorCode PlexLayerLoad_Private(PlexLayer layer, PetscViewer viewer, PetscInt d, PetscLayout pointsLayout)
2210: {
2211: char path[128];
2212: MPI_Comm comm;
2213: const char *coneSizesName, *conesName, *orientationsName;
2214: IS coneSizesIS, conesIS, orientationsIS;
2215: PetscSection coneSizesSection;
2216: PetscLayout vertexLayout = NULL;
2217: PetscInt s;
2219: PetscFunctionBegin;
2220: coneSizesName = "cone_sizes";
2221: conesName = "cones";
2222: orientationsName = "orientations";
2223: PetscCall(PetscObjectGetComm((PetscObject)viewer, &comm));
2225: /* query size of next lower depth stratum (next lower dimension) */
2226: if (d > 0) {
2227: PetscInt NVertices;
2229: PetscCall(PetscSNPrintf(path, sizeof(path), "%" PetscInt_FMT "/%s", d - 1, coneSizesName));
2230: PetscCall(PetscViewerHDF5ReadSizes(viewer, path, NULL, &NVertices));
2231: PetscCall(PetscLayoutCreate(comm, &vertexLayout));
2232: PetscCall(PetscLayoutSetSize(vertexLayout, NVertices));
2233: PetscCall(PetscLayoutSetUp(vertexLayout));
2234: }
2236: PetscCall(PetscSNPrintf(path, sizeof(path), "%" PetscInt_FMT, d));
2237: PetscCall(PetscViewerHDF5PushGroup(viewer, path));
2239: /* create coneSizesSection from stored IS coneSizes */
2240: {
2241: const PetscInt *coneSizes;
2243: PetscCall(ISCreate(comm, &coneSizesIS));
2244: PetscCall(PetscObjectSetName((PetscObject)coneSizesIS, coneSizesName));
2245: if (pointsLayout) PetscCall(ISSetLayout(coneSizesIS, pointsLayout));
2246: PetscCall(ISLoad(coneSizesIS, viewer));
2247: if (!pointsLayout) PetscCall(ISGetLayout(coneSizesIS, &pointsLayout));
2248: PetscCall(ISGetIndices(coneSizesIS, &coneSizes));
2249: PetscCall(PetscSectionCreate(comm, &coneSizesSection));
2250: //TODO different start ?
2251: PetscCall(PetscSectionSetChart(coneSizesSection, 0, pointsLayout->n));
2252: for (s = 0; s < pointsLayout->n; ++s) PetscCall(PetscSectionSetDof(coneSizesSection, s, coneSizes[s]));
2253: PetscCall(PetscSectionSetUp(coneSizesSection));
2254: PetscCall(ISRestoreIndices(coneSizesIS, &coneSizes));
2255: {
2256: PetscLayout tmp = NULL;
2258: /* We need to keep the layout until the end of function */
2259: PetscCall(PetscLayoutReference((PetscLayout)pointsLayout, &tmp));
2260: }
2261: PetscCall(ISDestroy(&coneSizesIS));
2262: }
2264: /* use value layout of coneSizesSection as layout of cones and orientations */
2265: {
2266: PetscLayout conesLayout;
2268: PetscCall(PetscSectionGetValueLayout(comm, coneSizesSection, &conesLayout));
2269: PetscCall(ISCreate(comm, &conesIS));
2270: PetscCall(ISCreate(comm, &orientationsIS));
2271: PetscCall(PetscObjectSetName((PetscObject)conesIS, conesName));
2272: PetscCall(PetscObjectSetName((PetscObject)orientationsIS, orientationsName));
2273: PetscCall(PetscLayoutDuplicate(conesLayout, &conesIS->map));
2274: PetscCall(PetscLayoutDuplicate(conesLayout, &orientationsIS->map));
2275: PetscCall(ISLoad(conesIS, viewer));
2276: PetscCall(ISLoad(orientationsIS, viewer));
2277: PetscCall(PetscLayoutDestroy(&conesLayout));
2278: }
2280: /* check assertion that layout of points is the same as point layout of coneSizesSection */
2281: {
2282: PetscLayout pointsLayout0;
2283: PetscBool flg;
2285: PetscCall(PetscSectionGetPointLayout(comm, coneSizesSection, &pointsLayout0));
2286: PetscCall(PetscLayoutCompare(pointsLayout, pointsLayout0, &flg));
2287: PetscCheck(flg, comm, PETSC_ERR_PLIB, "points layout != coneSizesSection point layout");
2288: PetscCall(PetscLayoutDestroy(&pointsLayout0));
2289: }
2290: PetscCall(PetscViewerHDF5PopGroup(viewer));
2291: PetscCall(PetscLayoutDestroy(&pointsLayout));
2293: layer->d = d;
2294: layer->conesIS = conesIS;
2295: layer->coneSizesSection = coneSizesSection;
2296: layer->orientationsIS = orientationsIS;
2297: layer->vertexLayout = vertexLayout;
2298: PetscFunctionReturn(PETSC_SUCCESS);
2299: }
2301: static PetscErrorCode PlexLayerDistribute_Private(PlexLayer layer, PetscSF cellLocalToGlobalSF)
2302: {
2303: IS newConesIS, newOrientationsIS;
2304: PetscSection newConeSizesSection;
2305: MPI_Comm comm;
2307: PetscFunctionBegin;
2308: PetscCall(PetscObjectGetComm((PetscObject)cellLocalToGlobalSF, &comm));
2309: PetscCall(PetscSectionCreate(comm, &newConeSizesSection));
2310: //TODO rename to something like ISDistribute() with optional PetscSection argument
2311: PetscCall(DMPlexDistributeFieldIS(NULL, cellLocalToGlobalSF, layer->coneSizesSection, layer->conesIS, newConeSizesSection, &newConesIS));
2312: PetscCall(DMPlexDistributeFieldIS(NULL, cellLocalToGlobalSF, layer->coneSizesSection, layer->orientationsIS, newConeSizesSection, &newOrientationsIS));
2314: PetscCall(PetscObjectSetName((PetscObject)newConeSizesSection, ((PetscObject)layer->coneSizesSection)->name));
2315: PetscCall(PetscObjectSetName((PetscObject)newConesIS, ((PetscObject)layer->conesIS)->name));
2316: PetscCall(PetscObjectSetName((PetscObject)newOrientationsIS, ((PetscObject)layer->orientationsIS)->name));
2317: PetscCall(PetscSectionDestroy(&layer->coneSizesSection));
2318: PetscCall(ISDestroy(&layer->conesIS));
2319: PetscCall(ISDestroy(&layer->orientationsIS));
2320: layer->coneSizesSection = newConeSizesSection;
2321: layer->conesIS = newConesIS;
2322: layer->orientationsIS = newOrientationsIS;
2323: PetscFunctionReturn(PETSC_SUCCESS);
2324: }
2326: //TODO share code with DMPlexBuildFromCellListParallel()
2327: #include <petsc/private/hashseti.h>
2328: static PetscErrorCode PlexLayerCreateSFs_Private(PlexLayer layer, PetscSF *vertexOverlapSF, PetscSF *sfXC)
2329: {
2330: PetscLayout vertexLayout = layer->vertexLayout;
2331: PetscSection coneSection = layer->coneSizesSection;
2332: IS cellVertexData = layer->conesIS;
2333: IS coneOrientations = layer->orientationsIS;
2334: PetscSF vl2gSF, vOverlapSF;
2335: PetscInt *verticesAdj;
2336: PetscInt i, n, numVerticesAdj;
2337: const PetscInt *cvd, *co = NULL;
2338: MPI_Comm comm;
2340: PetscFunctionBegin;
2341: PetscCall(PetscObjectGetComm((PetscObject)coneSection, &comm));
2342: PetscCall(PetscSectionGetStorageSize(coneSection, &n));
2343: {
2344: PetscInt n0;
2346: PetscCall(ISGetLocalSize(cellVertexData, &n0));
2347: PetscCheck(n == n0, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Local size of IS cellVertexData = %" PetscInt_FMT " != %" PetscInt_FMT " = storage size of PetscSection coneSection", n0, n);
2348: PetscCall(ISGetIndices(cellVertexData, &cvd));
2349: }
2350: if (coneOrientations) {
2351: PetscInt n0;
2353: PetscCall(ISGetLocalSize(coneOrientations, &n0));
2354: PetscCheck(n == n0, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Local size of IS coneOrientations = %" PetscInt_FMT " != %" PetscInt_FMT " = storage size of PetscSection coneSection", n0, n);
2355: PetscCall(ISGetIndices(coneOrientations, &co));
2356: }
2357: /* Get/check global number of vertices */
2358: {
2359: PetscInt NVerticesInCells = PETSC_INT_MIN;
2361: /* NVerticesInCells = max(cellVertexData) + 1 */
2362: for (i = 0; i < n; i++)
2363: if (cvd[i] > NVerticesInCells) NVerticesInCells = cvd[i];
2364: ++NVerticesInCells;
2365: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, comm));
2367: if (vertexLayout->n == PETSC_DECIDE && vertexLayout->N == PETSC_DECIDE) vertexLayout->N = NVerticesInCells;
2368: else
2369: PetscCheck(vertexLayout->N == PETSC_DECIDE || vertexLayout->N >= NVerticesInCells, comm, PETSC_ERR_ARG_SIZ, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of unique vertices in the cell-vertex dataset %" PetscInt_FMT,
2370: vertexLayout->N, NVerticesInCells);
2371: PetscCall(PetscLayoutSetUp(vertexLayout));
2372: }
2373: /* Find locally unique vertices in cellVertexData */
2374: {
2375: PetscHSetI vhash;
2376: PetscInt off = 0;
2378: PetscCall(PetscHSetICreate(&vhash));
2379: for (i = 0; i < n; ++i) PetscCall(PetscHSetIAdd(vhash, cvd[i]));
2380: PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
2381: PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
2382: PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
2383: PetscAssert(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Assertion failed: off == numVerticesAdj (%" PetscInt_FMT " != %" PetscInt_FMT ")", off, numVerticesAdj);
2384: PetscCall(PetscHSetIDestroy(&vhash));
2385: }
2386: /* We must sort vertices to preserve numbering */
2387: PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
2388: /* Connect locally unique vertices with star forest */
2389: PetscCall(PetscSFCreateByMatchingIndices(vertexLayout, numVerticesAdj, verticesAdj, NULL, 0, numVerticesAdj, verticesAdj, NULL, 0, &vl2gSF, &vOverlapSF));
2390: PetscCall(PetscObjectSetName((PetscObject)vOverlapSF, "overlapSF"));
2391: PetscCall(PetscObjectSetName((PetscObject)vl2gSF, "localToGlobalSF"));
2393: PetscCall(PetscFree(verticesAdj));
2394: *vertexOverlapSF = vOverlapSF;
2395: *sfXC = vl2gSF;
2396: PetscFunctionReturn(PETSC_SUCCESS);
2397: }
2399: static PetscErrorCode PlexLayerCreateCellSFs_Private(PlexLayer layer, PetscSF *cellOverlapSF, PetscSF *cellLocalToGlobalSF)
2400: {
2401: PetscSection coneSection = layer->coneSizesSection;
2402: PetscInt nCells;
2403: MPI_Comm comm;
2405: PetscFunctionBegin;
2406: PetscCall(PetscObjectGetComm((PetscObject)coneSection, &comm));
2407: {
2408: PetscInt cStart;
2410: PetscCall(PetscSectionGetChart(coneSection, &cStart, &nCells));
2411: PetscCheck(cStart == 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "coneSection must start at 0");
2412: }
2413: /* Create overlapSF as empty SF with the right number of roots */
2414: PetscCall(PetscSFCreate(comm, cellOverlapSF));
2415: PetscCall(PetscSFSetGraph(*cellOverlapSF, nCells, 0, NULL, PETSC_USE_POINTER, NULL, PETSC_USE_POINTER));
2416: PetscCall(PetscSFSetUp(*cellOverlapSF));
2417: /* Create localToGlobalSF as identity mapping */
2418: {
2419: PetscLayout map;
2421: PetscCall(PetscLayoutCreateFromSizes(comm, nCells, PETSC_DECIDE, 1, &map));
2422: PetscCall(PetscSFCreateFromLayouts(map, map, cellLocalToGlobalSF));
2423: PetscCall(PetscSFSetUp(*cellLocalToGlobalSF));
2424: PetscCall(PetscLayoutDestroy(&map));
2425: }
2426: PetscFunctionReturn(PETSC_SUCCESS);
2427: }
2429: static PetscErrorCode DMPlexTopologyBuildFromLayers_Private(DM dm, PetscInt depth, PlexLayer *layers, IS strataPermutation)
2430: {
2431: const PetscInt *permArr;
2432: PetscInt d, nPoints;
2433: MPI_Comm comm;
2435: PetscFunctionBegin;
2436: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2437: PetscCall(ISGetIndices(strataPermutation, &permArr));
2439: /* Count points, strata offsets and cones offsets (taking strataPermutation into account) */
2440: {
2441: PetscInt stratumOffset = 0;
2442: PetscInt conesOffset = 0;
2444: for (d = 0; d <= depth; d++) {
2445: const PetscInt e = permArr[d];
2446: const PlexLayer l = layers[e];
2447: PetscInt lo, n, size;
2449: PetscCall(PetscSectionGetChart(l->coneSizesSection, &lo, &n));
2450: PetscCall(PetscSectionGetStorageSize(l->coneSizesSection, &size));
2451: PetscCheck(lo == 0, comm, PETSC_ERR_PLIB, "starting point should be 0 in coneSizesSection %" PetscInt_FMT, d);
2452: l->offset = stratumOffset;
2453: l->conesOffset = conesOffset;
2454: stratumOffset += n;
2455: conesOffset += size;
2456: }
2457: nPoints = stratumOffset;
2458: }
2460: /* Set interval for all plex points */
2461: //TODO we should store starting point of plex
2462: PetscCall(DMPlexSetChart(dm, 0, nPoints));
2464: /* Set up plex coneSection from layer coneSections */
2465: {
2466: PetscSection coneSection;
2468: PetscCall(DMPlexGetConeSection(dm, &coneSection));
2469: for (d = 0; d <= depth; d++) {
2470: const PlexLayer l = layers[d];
2471: PetscInt n, q;
2473: PetscCall(PetscSectionGetChart(l->coneSizesSection, NULL, &n));
2474: for (q = 0; q < n; q++) {
2475: const PetscInt p = l->offset + q;
2476: PetscInt coneSize;
2478: PetscCall(PetscSectionGetDof(l->coneSizesSection, q, &coneSize));
2479: PetscCall(PetscSectionSetDof(coneSection, p, coneSize));
2480: }
2481: }
2482: }
2483: //TODO this is terrible, DMSetUp_Plex() should be DMPlexSetUpSections() or so
2484: PetscCall(DMSetUp(dm));
2486: /* Renumber cones points from layer-global numbering to plex-local numbering */
2487: {
2488: PetscInt *cones, *ornts;
2490: PetscCall(DMPlexGetCones(dm, &cones));
2491: PetscCall(DMPlexGetConeOrientations(dm, &ornts));
2492: for (d = 1; d <= depth; d++) {
2493: const PlexLayer l = layers[d];
2494: PetscInt i, lConesSize;
2495: PetscInt *lCones;
2496: const PetscInt *lOrnts;
2497: PetscInt *pCones = &cones[l->conesOffset];
2498: PetscInt *pOrnts = &ornts[l->conesOffset];
2500: PetscCall(PetscSectionGetStorageSize(l->coneSizesSection, &lConesSize));
2501: /* Get cones in local plex numbering */
2502: {
2503: ISLocalToGlobalMapping l2g;
2504: PetscLayout vertexLayout = l->vertexLayout;
2505: PetscSF vertexSF = layers[d - 1]->l2gSF; /* vertices of this layer are cells of previous layer */
2506: const PetscInt *gCones;
2507: PetscInt lConesSize0;
2509: PetscCall(ISGetLocalSize(l->conesIS, &lConesSize0));
2510: PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "layer %" PetscInt_FMT " size(conesIS) = %" PetscInt_FMT " != %" PetscInt_FMT " = storageSize(coneSizesSection)", d, lConesSize0, lConesSize);
2511: PetscCall(ISGetLocalSize(l->orientationsIS, &lConesSize0));
2512: PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "layer %" PetscInt_FMT " size(orientationsIS) = %" PetscInt_FMT " != %" PetscInt_FMT " = storageSize(coneSizesSection)", d, lConesSize0, lConesSize);
2514: PetscCall(PetscMalloc1(lConesSize, &lCones));
2515: PetscCall(ISGetIndices(l->conesIS, &gCones));
2516: PetscCall(ISLocalToGlobalMappingCreateSF(vertexSF, vertexLayout->rstart, &l2g));
2517: PetscCall(ISGlobalToLocalMappingApply(l2g, IS_GTOLM_MASK, lConesSize, gCones, &lConesSize0, lCones));
2518: PetscCheck(lConesSize0 == lConesSize, comm, PETSC_ERR_PLIB, "global to local does not cover all indices (%" PetscInt_FMT " of %" PetscInt_FMT ")", lConesSize0, lConesSize);
2519: PetscCall(ISLocalToGlobalMappingDestroy(&l2g));
2520: PetscCall(ISRestoreIndices(l->conesIS, &gCones));
2521: }
2522: PetscCall(ISGetIndices(l->orientationsIS, &lOrnts));
2523: /* Set cones, need to add stratum offset */
2524: for (i = 0; i < lConesSize; i++) {
2525: pCones[i] = lCones[i] + layers[d - 1]->offset; /* cone points of current layer are points of previous layer */
2526: pOrnts[i] = lOrnts[i];
2527: }
2528: PetscCall(PetscFree(lCones));
2529: PetscCall(ISRestoreIndices(l->orientationsIS, &lOrnts));
2530: }
2531: }
2532: PetscCall(DMPlexSymmetrize(dm));
2533: PetscCall(DMPlexStratify(dm));
2534: PetscCall(ISRestoreIndices(strataPermutation, &permArr));
2535: PetscFunctionReturn(PETSC_SUCCESS);
2536: }
2538: static PetscErrorCode PlexLayerConcatenateSFs_Private(MPI_Comm comm, PetscInt depth, PlexLayer layers[], IS strataPermutation, PetscSF *overlapSF, PetscSF *l2gSF)
2539: {
2540: PetscInt d;
2541: PetscSF *osfs, *lsfs;
2542: PetscInt *leafOffsets;
2543: const PetscInt *permArr;
2545: PetscFunctionBegin;
2546: PetscCall(ISGetIndices(strataPermutation, &permArr));
2547: PetscCall(PetscCalloc3(depth + 1, &osfs, depth + 1, &lsfs, depth + 1, &leafOffsets));
2548: for (d = 0; d <= depth; d++) {
2549: const PetscInt e = permArr[d];
2551: PetscAssert(e == layers[e]->d, PETSC_COMM_SELF, PETSC_ERR_PLIB, "assertion: e == layers[e]->d");
2552: osfs[d] = layers[e]->overlapSF;
2553: lsfs[d] = layers[e]->l2gSF;
2554: leafOffsets[d] = layers[e]->offset;
2555: }
2556: PetscCall(PetscSFConcatenate(comm, depth + 1, osfs, PETSCSF_CONCATENATE_ROOTMODE_LOCAL, leafOffsets, overlapSF));
2557: PetscCall(PetscSFConcatenate(comm, depth + 1, lsfs, PETSCSF_CONCATENATE_ROOTMODE_GLOBAL, leafOffsets, l2gSF));
2558: PetscCall(PetscFree3(osfs, lsfs, leafOffsets));
2559: PetscCall(ISRestoreIndices(strataPermutation, &permArr));
2560: PetscFunctionReturn(PETSC_SUCCESS);
2561: }
2563: // Parallel load of topology
2564: static PetscErrorCode DMPlexTopologyLoad_HDF5_Private(DM dm, PetscViewer viewer, PetscSF *sfXC)
2565: {
2566: PlexLayer *layers;
2567: IS strataPermutation;
2568: PetscLayout pointsLayout;
2569: PetscInt depth;
2570: PetscInt d;
2571: MPI_Comm comm;
2573: PetscFunctionBegin;
2574: {
2575: PetscInt dim;
2577: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "depth", PETSC_INT, NULL, &depth));
2578: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "cell_dim", PETSC_INT, NULL, &dim));
2579: PetscCall(DMSetDimension(dm, dim));
2580: }
2581: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2583: PetscCall(PetscInfo(dm, "Loading DM %s in parallel\n", dm->hdr.name));
2584: {
2585: IS spOnComm;
2587: PetscCall(ISCreate(comm, &spOnComm));
2588: PetscCall(PetscObjectSetName((PetscObject)spOnComm, "permutation"));
2589: PetscCall(ISLoad(spOnComm, viewer));
2590: /* have the same serial IS on every rank */
2591: PetscCall(ISAllGather(spOnComm, &strataPermutation));
2592: PetscCall(PetscObjectSetName((PetscObject)strataPermutation, ((PetscObject)spOnComm)->name));
2593: PetscCall(ISDestroy(&spOnComm));
2594: }
2596: /* Create layers, load raw data for each layer */
2597: PetscCall(PetscViewerHDF5PushGroup(viewer, "strata"));
2598: PetscCall(PetscMalloc1(depth + 1, &layers));
2599: for (d = depth, pointsLayout = NULL; d >= 0; pointsLayout = layers[d]->vertexLayout, d--) {
2600: PetscCall(PlexLayerCreate_Private(&layers[d]));
2601: PetscCall(PlexLayerLoad_Private(layers[d], viewer, d, pointsLayout));
2602: }
2603: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* strata */
2605: for (d = depth; d >= 0; d--) {
2606: /* Redistribute cells and vertices for each applicable layer */
2607: if (d < depth) PetscCall(PlexLayerDistribute_Private(layers[d], layers[d]->l2gSF));
2608: /* Create vertex overlap SF and vertex localToGlobal SF for each applicable layer */
2609: if (d > 0) PetscCall(PlexLayerCreateSFs_Private(layers[d], &layers[d - 1]->overlapSF, &layers[d - 1]->l2gSF));
2610: }
2611: /* Build trivial SFs for the cell layer as well */
2612: PetscCall(PlexLayerCreateCellSFs_Private(layers[depth], &layers[depth]->overlapSF, &layers[depth]->l2gSF));
2614: /* Build DMPlex topology from the layers */
2615: PetscCall(DMPlexTopologyBuildFromLayers_Private(dm, depth, layers, strataPermutation));
2617: /* Build overall point SF alias overlap SF */
2618: {
2619: PetscSF overlapSF;
2621: PetscCall(PlexLayerConcatenateSFs_Private(comm, depth, layers, strataPermutation, &overlapSF, sfXC));
2622: PetscCall(DMSetPointSF(dm, overlapSF));
2623: PetscCall(PetscSFDestroy(&overlapSF));
2624: }
2626: for (d = depth; d >= 0; d--) PetscCall(PlexLayerDestroy(&layers[d]));
2627: PetscCall(PetscFree(layers));
2628: PetscCall(ISDestroy(&strataPermutation));
2629: PetscFunctionReturn(PETSC_SUCCESS);
2630: }
2632: PetscErrorCode DMPlexTopologyLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF *sfXC)
2633: {
2634: DMPlexStorageVersion version;
2635: const char *topologydm_name;
2636: char group[PETSC_MAX_PATH_LEN];
2637: PetscSF sfwork = NULL;
2639: PetscFunctionBegin;
2640: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2641: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2642: if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
2643: PetscCall(PetscSNPrintf(group, sizeof(group), "topologies/%s", topologydm_name));
2644: } else {
2645: PetscCall(PetscStrncpy(group, "/", sizeof(group)));
2646: }
2647: PetscCall(PetscViewerHDF5PushGroup(viewer, group));
2649: PetscCall(PetscViewerHDF5PushGroup(viewer, "topology"));
2650: if (version->major < 3) {
2651: PetscCall(DMPlexTopologyLoad_HDF5_Legacy_Private(dm, viewer, &sfwork));
2652: } else {
2653: /* since DMPlexStorageVersion 3.0.0 */
2654: PetscCall(DMPlexTopologyLoad_HDF5_Private(dm, viewer, &sfwork));
2655: }
2656: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "topology" */
2658: if (DMPlexStorageVersionGE(version, 2, 0, 0)) {
2659: DM distdm;
2660: PetscSF distsf;
2661: const char *distribution_name;
2662: PetscBool exists;
2664: PetscCall(DMPlexDistributionGetName(dm, &distribution_name));
2665: /* this group is guaranteed to be present since DMPlexStorageVersion 2.1.0 */
2666: PetscCall(PetscViewerHDF5PushGroup(viewer, "distributions"));
2667: PetscCall(PetscViewerHDF5HasGroup(viewer, NULL, &exists));
2668: if (exists) {
2669: PetscCall(PetscViewerHDF5PushGroup(viewer, distribution_name));
2670: PetscCall(DMPlexDistributionLoad_HDF5_Private(dm, viewer, sfwork, &distsf, &distdm));
2671: if (distdm) {
2672: PetscCall(DMPlexReplace_Internal(dm, &distdm));
2673: PetscCall(PetscSFDestroy(&sfwork));
2674: sfwork = distsf;
2675: }
2676: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* distribution_name */
2677: }
2678: PetscCall(PetscViewerHDF5PopGroup(viewer)); /* "distributions" */
2679: }
2680: if (sfXC) {
2681: *sfXC = sfwork;
2682: } else {
2683: PetscCall(PetscSFDestroy(&sfwork));
2684: }
2686: PetscCall(PetscViewerHDF5PopGroup(viewer));
2687: PetscFunctionReturn(PETSC_SUCCESS);
2688: }
2690: /* If the file is old, it not only has different path to the coordinates, but */
2691: /* does not contain coordinateDMs, so must fall back to the old implementation. */
2692: static PetscErrorCode DMPlexCoordinatesLoad_HDF5_Legacy_Private(DM dm, PetscViewer viewer)
2693: {
2694: PetscSection coordSection;
2695: Vec coordinates;
2696: PetscReal lengthScale;
2697: PetscInt spatialDim, N, numVertices, vStart, vEnd, v;
2698: PetscMPIInt rank;
2700: PetscFunctionBegin;
2701: PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2702: /* Read geometry */
2703: PetscCall(PetscViewerHDF5PushGroup(viewer, "/geometry"));
2704: PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
2705: PetscCall(PetscObjectSetName((PetscObject)coordinates, "vertices"));
2706: {
2707: /* Force serial load */
2708: PetscCall(PetscViewerHDF5ReadSizes(viewer, "vertices", &spatialDim, &N));
2709: PetscCall(VecSetSizes(coordinates, rank == 0 ? N : 0, N));
2710: PetscCall(VecSetBlockSize(coordinates, spatialDim));
2711: }
2712: PetscCall(VecLoad(coordinates, viewer));
2713: PetscCall(PetscViewerHDF5PopGroup(viewer));
2714: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
2715: PetscCall(VecScale(coordinates, 1.0 / lengthScale));
2716: PetscCall(VecGetLocalSize(coordinates, &numVertices));
2717: PetscCall(VecGetBlockSize(coordinates, &spatialDim));
2718: numVertices /= spatialDim;
2719: /* Create coordinates */
2720: PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2721: PetscCheck(numVertices == vEnd - vStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coordinates loaded %" PetscInt_FMT " does not match number of vertices %" PetscInt_FMT, numVertices, vEnd - vStart);
2722: PetscCall(DMGetCoordinateSection(dm, &coordSection));
2723: PetscCall(PetscSectionSetNumFields(coordSection, 1));
2724: PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spatialDim));
2725: PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
2726: for (v = vStart; v < vEnd; ++v) {
2727: PetscCall(PetscSectionSetDof(coordSection, v, spatialDim));
2728: PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spatialDim));
2729: }
2730: PetscCall(PetscSectionSetUp(coordSection));
2731: PetscCall(DMSetCoordinates(dm, coordinates));
2732: PetscCall(VecDestroy(&coordinates));
2733: PetscFunctionReturn(PETSC_SUCCESS);
2734: }
2736: PetscErrorCode DMPlexCoordinatesLoad_HDF5_Internal(DM dm, PetscViewer viewer, PetscSF sfXC)
2737: {
2738: DMPlexStorageVersion version;
2739: DM cdm;
2740: Vec coords;
2741: PetscInt blockSize;
2742: PetscReal lengthScale;
2743: PetscSF lsf;
2744: const char *topologydm_name;
2745: char *coordinatedm_name, *coordinates_name;
2747: PetscFunctionBegin;
2748: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2749: if (!DMPlexStorageVersionGE(version, 2, 0, 0)) {
2750: PetscCall(DMPlexCoordinatesLoad_HDF5_Legacy_Private(dm, viewer));
2751: PetscFunctionReturn(PETSC_SUCCESS);
2752: }
2753: /* else: since DMPlexStorageVersion 2.0.0 */
2754: PetscCheck(sfXC, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_NULL, "PetscSF must be given for parallel load");
2755: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2756: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
2757: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
2758: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "coordinateDMName", PETSC_STRING, NULL, &coordinatedm_name));
2759: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "coordinatesName", PETSC_STRING, NULL, &coordinates_name));
2760: PetscCall(PetscViewerHDF5PopGroup(viewer));
2761: PetscCall(PetscViewerHDF5PopGroup(viewer));
2762: PetscCall(DMGetCoordinateDM(dm, &cdm));
2763: PetscCall(PetscObjectSetName((PetscObject)cdm, coordinatedm_name));
2764: PetscCall(PetscFree(coordinatedm_name));
2765: /* lsf: on-disk data -> in-memory local vector associated with cdm's local section */
2766: PetscCall(DMPlexSectionLoad(dm, viewer, cdm, sfXC, NULL, &lsf));
2767: PetscCall(DMCreateLocalVector(cdm, &coords));
2768: PetscCall(PetscObjectSetName((PetscObject)coords, coordinates_name));
2769: PetscCall(PetscFree(coordinates_name));
2770: PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE));
2771: PetscCall(DMPlexLocalVectorLoad(dm, viewer, cdm, lsf, coords));
2772: PetscCall(PetscViewerPopFormat(viewer));
2773: PetscCall(DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale));
2774: PetscCall(VecScale(coords, 1.0 / lengthScale));
2775: PetscCall(DMSetCoordinatesLocal(dm, coords));
2776: PetscCall(VecGetBlockSize(coords, &blockSize));
2777: PetscCall(DMSetCoordinateDim(dm, blockSize));
2778: PetscCall(VecDestroy(&coords));
2779: PetscCall(PetscSFDestroy(&lsf));
2780: PetscFunctionReturn(PETSC_SUCCESS);
2781: }
2783: PetscErrorCode DMPlexLoad_HDF5_Internal(DM dm, PetscViewer viewer)
2784: {
2785: DMPlexStorageVersion version;
2787: PetscFunctionBegin;
2788: PetscCall(PetscViewerHDF5GetDMPlexStorageVersionReading(viewer, &version));
2789: PetscCall(PetscInfo(dm, "Loading DM %s storage version %d.%d.%d\n", dm->hdr.name, version->major, version->minor, version->subminor));
2790: if (!DMPlexStorageVersionGE(version, 2, 0, 0)) {
2791: PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, NULL));
2792: PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, NULL));
2793: PetscCall(DMPlexCoordinatesLoad_HDF5_Legacy_Private(dm, viewer));
2794: } else {
2795: PetscSF sfXC;
2797: /* since DMPlexStorageVersion 2.0.0 */
2798: PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, &sfXC));
2799: PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, sfXC));
2800: PetscCall(DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, sfXC));
2801: PetscCall(PetscSFDestroy(&sfXC));
2802: }
2803: PetscFunctionReturn(PETSC_SUCCESS);
2804: }
2806: static PetscErrorCode DMPlexSectionLoad_HDF5_Internal_CreateDataSF(PetscSection rootSection, PetscLayout layout, PetscInt globalOffsets[], PetscSection leafSection, PetscSF *sectionSF)
2807: {
2808: MPI_Comm comm;
2809: PetscInt pStart, pEnd, p, m;
2810: PetscInt *goffs, *ilocal;
2811: PetscBool rootIncludeConstraints, leafIncludeConstraints;
2813: PetscFunctionBegin;
2814: PetscCall(PetscObjectGetComm((PetscObject)leafSection, &comm));
2815: PetscCall(PetscSectionGetChart(leafSection, &pStart, &pEnd));
2816: PetscCall(PetscSectionGetIncludesConstraints(rootSection, &rootIncludeConstraints));
2817: PetscCall(PetscSectionGetIncludesConstraints(leafSection, &leafIncludeConstraints));
2818: if (rootIncludeConstraints && leafIncludeConstraints) PetscCall(PetscSectionGetStorageSize(leafSection, &m));
2819: else PetscCall(PetscSectionGetConstrainedStorageSize(leafSection, &m));
2820: PetscCall(PetscMalloc1(m, &ilocal));
2821: PetscCall(PetscMalloc1(m, &goffs));
2822: /* Currently, PetscSFDistributeSection() returns globalOffsets[] only */
2823: /* for the top-level section (not for each field), so one must have */
2824: /* rootSection->pointMajor == PETSC_TRUE. */
2825: PetscCheck(rootSection->pointMajor, PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for field major ordering");
2826: /* Currently, we also assume that leafSection->pointMajor == PETSC_TRUE. */
2827: PetscCheck(leafSection->pointMajor, PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for field major ordering");
2828: for (p = pStart, m = 0; p < pEnd; ++p) {
2829: PetscInt dof, cdof, i, j, off, goff;
2830: const PetscInt *cinds;
2832: PetscCall(PetscSectionGetDof(leafSection, p, &dof));
2833: if (dof < 0) continue;
2834: goff = globalOffsets[p - pStart];
2835: PetscCall(PetscSectionGetOffset(leafSection, p, &off));
2836: PetscCall(PetscSectionGetConstraintDof(leafSection, p, &cdof));
2837: PetscCall(PetscSectionGetConstraintIndices(leafSection, p, &cinds));
2838: for (i = 0, j = 0; i < dof; ++i) {
2839: PetscBool constrained = (PetscBool)(j < cdof && i == cinds[j]);
2841: if (!constrained || (leafIncludeConstraints && rootIncludeConstraints)) {
2842: ilocal[m] = off++;
2843: goffs[m++] = goff++;
2844: } else if (leafIncludeConstraints && !rootIncludeConstraints) ++off;
2845: else if (!leafIncludeConstraints && rootIncludeConstraints) ++goff;
2846: if (constrained) ++j;
2847: }
2848: }
2849: PetscCall(PetscSFCreate(comm, sectionSF));
2850: PetscCall(PetscSFSetFromOptions(*sectionSF));
2851: PetscCall(PetscSFSetGraphLayout(*sectionSF, layout, m, ilocal, PETSC_OWN_POINTER, goffs));
2852: PetscCall(PetscFree(goffs));
2853: PetscFunctionReturn(PETSC_SUCCESS);
2854: }
2856: PetscErrorCode DMPlexSectionLoad_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sfXB, PetscSF *gsf, PetscSF *lsf)
2857: {
2858: MPI_Comm comm;
2859: PetscMPIInt size, rank;
2860: const char *topologydm_name;
2861: const char *sectiondm_name;
2862: PetscSection sectionA, sectionB;
2863: PetscBool has;
2864: PetscInt nX, n, i;
2865: PetscSF sfAB;
2867: PetscFunctionBegin;
2868: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2869: PetscCallMPI(MPI_Comm_size(comm, &size));
2870: PetscCallMPI(MPI_Comm_rank(comm, &rank));
2871: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
2872: PetscCall(PetscObjectGetName((PetscObject)sectiondm, §iondm_name));
2873: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
2874: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
2875: PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
2876: PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
2877: /* A: on-disk points */
2878: /* X: list of global point numbers, [0, NX) */
2879: /* B: plex points */
2880: /* Load raw section (sectionA) */
2881: PetscCall(PetscSectionCreate(comm, §ionA));
2882: PetscCall(PetscViewerHDF5HasGroup(viewer, "section", &has));
2883: if (has) PetscCall(PetscSectionLoad(sectionA, viewer));
2884: else {
2885: // TODO If section is missing, create the default affine section with dim dofs on each vertex. Use PetscSplitOwnership() to split vertices.
2886: // How do I know the total number of vertices?
2887: PetscInt dim, Nf = 1, Nv, nv = PETSC_DECIDE;
2889: PetscCall(DMGetDimension(dm, &dim));
2890: PetscCall(DMPlexGetDepthStratumGlobalSize(dm, 0, &Nv));
2891: PetscCall(PetscSectionSetNumFields(sectionA, Nf));
2892: PetscCall(PetscSectionSetFieldName(sectionA, 0, "Cartesian"));
2893: PetscCall(PetscSectionSetFieldComponents(sectionA, 0, dim));
2894: for (PetscInt c = 0; c < dim; ++c) {
2895: char axis = 'X' + (char)c;
2897: PetscCall(PetscSectionSetComponentName(sectionA, 0, c, &axis));
2898: }
2899: PetscCall(PetscSplitOwnership(comm, &nv, &Nv));
2900: PetscCall(PetscSectionSetChart(sectionA, 0, nv));
2901: for (PetscInt p = 0; p < nv; ++p) {
2902: PetscCall(PetscSectionSetDof(sectionA, p, dim));
2903: PetscCall(PetscSectionSetFieldDof(sectionA, p, 0, dim));
2904: }
2905: PetscCall(PetscSectionSetUp(sectionA));
2906: }
2907: PetscCall(PetscSectionGetChart(sectionA, NULL, &n));
2908: /* Create sfAB: A -> B */
2909: #if defined(PETSC_USE_DEBUG)
2910: {
2911: PetscInt N, N1;
2913: PetscCall(PetscViewerHDF5ReadSizes(viewer, "order", NULL, &N1));
2914: PetscCallMPI(MPIU_Allreduce(&n, &N, 1, MPIU_INT, MPI_SUM, comm));
2915: PetscCheck(N1 == N, comm, PETSC_ERR_ARG_SIZ, "Mismatching sizes: on-disk order array size (%" PetscInt_FMT ") != number of loaded section points (%" PetscInt_FMT ")", N1, N);
2916: }
2917: #endif
2918: {
2919: IS orderIS;
2920: const PetscInt *gpoints;
2921: PetscSF sfXA, sfAX;
2922: PetscLayout layout;
2923: PetscSFNode *owners, *buffer;
2924: PetscInt nleaves;
2925: PetscInt *ilocal;
2926: PetscSFNode *iremote;
2928: /* Create sfAX: A -> X */
2929: PetscCall(ISCreate(comm, &orderIS));
2930: PetscCall(PetscObjectSetName((PetscObject)orderIS, "order"));
2931: PetscCall(PetscLayoutSetLocalSize(orderIS->map, n));
2932: PetscCall(ISLoad(orderIS, viewer));
2933: PetscCall(PetscLayoutCreate(comm, &layout));
2934: PetscCall(PetscSFGetGraph(sfXB, &nX, NULL, NULL, NULL));
2935: PetscCall(PetscLayoutSetLocalSize(layout, nX));
2936: PetscCall(PetscLayoutSetBlockSize(layout, 1));
2937: PetscCall(PetscLayoutSetUp(layout));
2938: PetscCall(PetscSFCreate(comm, &sfXA));
2939: PetscCall(ISGetIndices(orderIS, &gpoints));
2940: PetscCall(PetscSFSetGraphLayout(sfXA, layout, n, NULL, PETSC_OWN_POINTER, gpoints));
2941: PetscCall(ISRestoreIndices(orderIS, &gpoints));
2942: PetscCall(ISDestroy(&orderIS));
2943: PetscCall(PetscLayoutDestroy(&layout));
2944: PetscCall(PetscMalloc1(n, &owners));
2945: PetscCall(PetscMalloc1(nX, &buffer));
2946: for (i = 0; i < n; ++i) {
2947: owners[i].rank = rank;
2948: owners[i].index = i;
2949: }
2950: for (i = 0; i < nX; ++i) {
2951: buffer[i].rank = -1;
2952: buffer[i].index = -1;
2953: }
2954: PetscCall(PetscSFReduceBegin(sfXA, MPIU_SF_NODE, owners, buffer, MPI_MAXLOC));
2955: PetscCall(PetscSFReduceEnd(sfXA, MPIU_SF_NODE, owners, buffer, MPI_MAXLOC));
2956: PetscCall(PetscSFDestroy(&sfXA));
2957: PetscCall(PetscFree(owners));
2958: for (i = 0, nleaves = 0; i < nX; ++i)
2959: if (buffer[i].rank >= 0) nleaves++;
2960: PetscCall(PetscMalloc1(nleaves, &ilocal));
2961: PetscCall(PetscMalloc1(nleaves, &iremote));
2962: for (i = 0, nleaves = 0; i < nX; ++i) {
2963: if (buffer[i].rank >= 0) {
2964: ilocal[nleaves] = i;
2965: iremote[nleaves].rank = buffer[i].rank;
2966: iremote[nleaves].index = buffer[i].index;
2967: nleaves++;
2968: }
2969: }
2970: PetscCall(PetscFree(buffer));
2971: PetscCall(PetscSFCreate(comm, &sfAX));
2972: PetscCall(PetscSFSetFromOptions(sfAX));
2973: PetscCall(PetscSFSetGraph(sfAX, n, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
2974: PetscCall(PetscSFCompose(sfAX, sfXB, &sfAB));
2975: PetscCall(PetscSFDestroy(&sfAX));
2976: }
2977: PetscCall(PetscViewerHDF5PopGroup(viewer));
2978: PetscCall(PetscViewerHDF5PopGroup(viewer));
2979: PetscCall(PetscViewerHDF5PopGroup(viewer));
2980: PetscCall(PetscViewerHDF5PopGroup(viewer));
2981: /* Create plex section (sectionB) */
2982: PetscCall(DMGetLocalSection(sectiondm, §ionB));
2983: if (lsf || gsf) {
2984: PetscLayout layout;
2985: PetscInt M, m;
2986: PetscInt *offsetsA;
2987: PetscBool includesConstraintsA;
2989: PetscCall(PetscSFDistributeSection(sfAB, sectionA, &offsetsA, sectionB));
2990: PetscCall(PetscSectionGetIncludesConstraints(sectionA, &includesConstraintsA));
2991: if (includesConstraintsA) PetscCall(PetscSectionGetStorageSize(sectionA, &m));
2992: else PetscCall(PetscSectionGetConstrainedStorageSize(sectionA, &m));
2993: PetscCallMPI(MPIU_Allreduce(&m, &M, 1, MPIU_INT, MPI_SUM, comm));
2994: PetscCall(PetscLayoutCreate(comm, &layout));
2995: PetscCall(PetscLayoutSetSize(layout, M));
2996: PetscCall(PetscLayoutSetUp(layout));
2997: if (lsf) {
2998: PetscSF lsfABdata;
3000: PetscCall(DMPlexSectionLoad_HDF5_Internal_CreateDataSF(sectionA, layout, offsetsA, sectionB, &lsfABdata));
3001: *lsf = lsfABdata;
3002: }
3003: if (gsf) {
3004: PetscSection gsectionB, gsectionB1;
3005: PetscBool includesConstraintsB;
3006: PetscSF gsfABdata, pointsf;
3008: PetscCall(DMGetGlobalSection(sectiondm, &gsectionB1));
3009: PetscCall(PetscSectionGetIncludesConstraints(gsectionB1, &includesConstraintsB));
3010: PetscCall(DMGetPointSF(sectiondm, &pointsf));
3011: PetscCall(PetscSectionCreateGlobalSection(sectionB, pointsf, PETSC_TRUE, includesConstraintsB, PETSC_TRUE, &gsectionB));
3012: PetscCall(DMPlexSectionLoad_HDF5_Internal_CreateDataSF(sectionA, layout, offsetsA, gsectionB, &gsfABdata));
3013: PetscCall(PetscSectionDestroy(&gsectionB));
3014: *gsf = gsfABdata;
3015: }
3016: PetscCall(PetscLayoutDestroy(&layout));
3017: PetscCall(PetscFree(offsetsA));
3018: } else {
3019: PetscCall(PetscSFDistributeSection(sfAB, sectionA, NULL, sectionB));
3020: }
3021: PetscCall(PetscSFDestroy(&sfAB));
3022: PetscCall(PetscSectionDestroy(§ionA));
3023: PetscFunctionReturn(PETSC_SUCCESS);
3024: }
3026: PetscErrorCode DMPlexVecLoad_HDF5_Internal(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
3027: {
3028: MPI_Comm comm;
3029: const char *topologydm_name;
3030: const char *sectiondm_name;
3031: const char *vec_name;
3032: Vec vecA;
3033: PetscInt mA, m, bs;
3034: const PetscInt *ilocal;
3035: const PetscScalar *src;
3036: PetscScalar *dest;
3038: PetscFunctionBegin;
3039: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3040: PetscCall(DMPlexGetHDF5Name_Private(dm, &topologydm_name));
3041: PetscCall(PetscObjectGetName((PetscObject)sectiondm, §iondm_name));
3042: PetscCall(PetscObjectGetName((PetscObject)vec, &vec_name));
3043: PetscCall(PetscViewerHDF5PushGroup(viewer, "topologies"));
3044: PetscCall(PetscViewerHDF5PushGroup(viewer, topologydm_name));
3045: PetscCall(PetscViewerHDF5PushGroup(viewer, "dms"));
3046: PetscCall(PetscViewerHDF5PushGroup(viewer, sectiondm_name));
3047: PetscCall(PetscViewerHDF5PushGroup(viewer, "vecs"));
3048: PetscCall(PetscViewerHDF5PushGroup(viewer, vec_name));
3049: PetscCall(VecCreate(comm, &vecA));
3050: PetscCall(PetscObjectSetName((PetscObject)vecA, vec_name));
3051: PetscCall(PetscSFGetGraph(sf, &mA, &m, &ilocal, NULL));
3052: /* Check consistency */
3053: {
3054: PetscSF pointsf, pointsf1;
3055: PetscInt m1, i, j;
3057: PetscCall(DMGetPointSF(dm, &pointsf));
3058: PetscCall(DMGetPointSF(sectiondm, &pointsf1));
3059: PetscCheck(pointsf1 == pointsf, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatching point SFs for dm and sectiondm");
3060: #if defined(PETSC_USE_DEBUG)
3061: {
3062: PetscInt MA, MA1;
3064: PetscCallMPI(MPIU_Allreduce(&mA, &MA, 1, MPIU_INT, MPI_SUM, comm));
3065: PetscCall(PetscViewerHDF5ReadSizes(viewer, vec_name, NULL, &MA1));
3066: PetscCheck(MA1 == MA, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Total SF root size (%" PetscInt_FMT ") != On-disk vector data size (%" PetscInt_FMT ")", MA, MA1);
3067: }
3068: #endif
3069: PetscCall(VecGetLocalSize(vec, &m1));
3070: PetscCheck(m1 >= m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Target vector size (%" PetscInt_FMT ") < SF leaf size (%" PetscInt_FMT ")", m1, m);
3071: for (i = 0; i < m; ++i) {
3072: j = ilocal ? ilocal[i] : i;
3073: PetscCheck(j >= 0 && j < m1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Leaf's %" PetscInt_FMT "-th index, %" PetscInt_FMT ", not in [%" PetscInt_FMT ", %" PetscInt_FMT ")", i, j, (PetscInt)0, m1);
3074: }
3075: }
3076: PetscCall(VecSetSizes(vecA, mA, PETSC_DECIDE));
3077: PetscCall(VecLoad(vecA, viewer));
3078: PetscCall(VecGetArrayRead(vecA, &src));
3079: PetscCall(VecGetArray(vec, &dest));
3080: PetscCall(PetscSFBcastBegin(sf, MPIU_SCALAR, src, dest, MPI_REPLACE));
3081: PetscCall(PetscSFBcastEnd(sf, MPIU_SCALAR, src, dest, MPI_REPLACE));
3082: PetscCall(VecRestoreArray(vec, &dest));
3083: PetscCall(VecRestoreArrayRead(vecA, &src));
3084: PetscCall(VecDestroy(&vecA));
3085: PetscCall(PetscViewerHDF5ReadAttribute(viewer, NULL, "blockSize", PETSC_INT, NULL, (void *)&bs));
3086: PetscCall(VecSetBlockSize(vec, bs));
3087: PetscCall(PetscViewerHDF5PopGroup(viewer));
3088: PetscCall(PetscViewerHDF5PopGroup(viewer));
3089: PetscCall(PetscViewerHDF5PopGroup(viewer));
3090: PetscCall(PetscViewerHDF5PopGroup(viewer));
3091: PetscCall(PetscViewerHDF5PopGroup(viewer));
3092: PetscCall(PetscViewerHDF5PopGroup(viewer));
3093: PetscFunctionReturn(PETSC_SUCCESS);
3094: }
3095: #endif