1: #pragma once

  3: /* MANSEC = Vec */
  4: /* SUBMANSEC = PetscSF */

  6: /*S
  7:    PetscSF - PETSc object for managing the communication of certain entries of arrays and `Vec` between MPI processes.

  9:    Level: intermediate

 11:   `PetscSF` uses the concept of star forests to indicate and determine the communication patterns concisely and efficiently.
 12:   A star  <https://en.wikipedia.org/wiki/Star_(graph_theory)> forest is simply a collection of trees of height 1. The leave nodes represent
 13:   "ghost locations" for the root nodes.

 15:   The standard usage paradigm for `PetscSF` is to provide the communication pattern with `PetscSFSetGraph()` or `PetscSFSetGraphWithPattern()` and
 16:   then perform the communication using `PetscSFBcastBegin()` and `PetscSFBcastEnd()`, `PetscSFReduceBegin()` and `PetscSFReduceEnd()`.

 18: .seealso: [](sec_petscsf), `PetscSFCreate()`, `PetscSFSetGraph()`, `PetscSFSetGraphWithPattern()`, `PetscSFBcastBegin()`, `PetscSFBcastEnd()`,
 19:           `PetscSFReduceBegin()`, `PetscSFReduceEnd()`, `VecScatter`, `VecScatterCreate()`
 20: S*/
 21: typedef struct _p_PetscSF *PetscSF;

 23: /*J
 24:   PetscSFType - String with the name of a `PetscSF` type. Each `PetscSFType` uses different mechanisms to perform the communication.

 26:   Level: beginner

 28:   Available Types:
 29: + `PETSCSFBASIC`      - use MPI sends and receives
 30: . `PETSCSFNEIGHBOR`   - use MPI_Neighbor operations
 31: . `PETSCSFALLGATHERV` - use MPI_Allgatherv operations
 32: . `PETSCSFALLGATHER`  - use MPI_Allgather operations
 33: . `PETSCSFGATHERV`    - use MPI_Igatherv and MPI_Iscatterv operations
 34: . `PETSCSFGATHER`     - use MPI_Igather and MPI_Iscatter operations
 35: . `PETSCSFALLTOALL`   - use MPI_Ialltoall operations
 36: - `PETSCSFWINDOW`     - use MPI_Win operations

 38:   Note:
 39:   Some `PetscSFType` only provide specialized code for a subset of the `PetscSF` operations and use `PETSCSFBASIC` for the others.

 41: .seealso: [](sec_petscsf), `PetscSFSetType()`, `PetscSF`
 42: J*/
 43: typedef const char *PetscSFType;
 44: #define PETSCSFBASIC      "basic"
 45: #define PETSCSFNEIGHBOR   "neighbor"
 46: #define PETSCSFALLGATHERV "allgatherv"
 47: #define PETSCSFALLGATHER  "allgather"
 48: #define PETSCSFGATHERV    "gatherv"
 49: #define PETSCSFGATHER     "gather"
 50: #define PETSCSFALLTOALL   "alltoall"
 51: #define PETSCSFWINDOW     "window"

 53: /*S
 54:    PetscSFNode - specifier of MPI rank owner and local index for array or `Vec` entry locations that are to be communicated with a `PetscSF`

 56:    Level: beginner

 58:   Sample Usage:
 59: .vb
 60:     PetscSFNode    *remote;
 61:     PetscCall(PetscMalloc1(nleaves,&remote));
 62:     for (i=0; i<size; i++) {
 63:       remote[i].rank = i;
 64:       remote[i].index = rank;
 65:     }
 66: .ve

 68:   Sample Fortran Usage:
 69: .vb
 70:     type(PetscSFNode) remote(6)
 71:     remote(1)%rank  = modulo(rank+size-1,size)
 72:     remote(1)%index = 1 * stride
 73: .ve

 75:   Notes:
 76:   Use  `MPIU_SF_NODE` when performing MPI operations on arrays of `PetscSFNode`

 78:   Generally the values of `rank` should be in $[ 0,size)$  and the value of `index` greater than or equal to 0, but there are some situations that violate this.

 80: .seealso: [](sec_petscsf), `PetscSF`, `PetscSFSetGraph()`
 81: S*/
 82: typedef struct {
 83:   PetscInt rank;  /* MPI rank of owner */
 84:   PetscInt index; /* Index of node on rank */
 85: } PetscSFNode;

 87: #define MPIU_SF_NODE MPIU_2INT

 89: /*E
 90:    PetscSFDirection - Direction in which a `PetscSF` communication is performed

 92:    Values:
 93: +   `PETSCSF_ROOT2LEAF` - data flows from roots to leaves (broadcast direction)
 94: -   `PETSCSF_LEAF2ROOT` - data flows from leaves to roots (reduce/fetch direction)

 96:    Level: developer

 98: .seealso: [](sec_petscsf), `PetscSF`, `PetscSFOperation`, `PetscSFBcastBegin()`, `PetscSFReduceBegin()`
 99: E*/
100: typedef enum {
101:   PETSCSF_ROOT2LEAF = 0,
102:   PETSCSF_LEAF2ROOT = 1
103: } PetscSFDirection;

105: /*E
106:    PetscSFOperation - Identifies the high-level operation being performed by a `PetscSF` communication

108:    Values:
109: +   `PETSCSF_BCAST`  - broadcast from roots to leaves
110: .   `PETSCSF_REDUCE` - reduce from leaves to roots with an `MPI_Op`
111: -   `PETSCSF_FETCH`  - fetch-and-op: each leaf receives the current root value and contributes its own

113:    Level: developer

115: .seealso: [](sec_petscsf), `PetscSF`, `PetscSFDirection`, `PetscSFBcastBegin()`, `PetscSFReduceBegin()`, `PetscSFFetchAndOpBegin()`
116: E*/
117: typedef enum {
118:   PETSCSF_BCAST  = 0,
119:   PETSCSF_REDUCE = 1,
120:   PETSCSF_FETCH  = 2
121: } PetscSFOperation;

123: /*E
124:    PetscSFBackend - Device backend used by a `PetscSF` to pack, unpack, and exchange data when doing device-aware communication

126:    Values:
127: +   `PETSCSF_BACKEND_INVALID` - no backend has been selected (the default for a host-only SF)
128: .   `PETSCSF_BACKEND_CUDA`    - use CUDA-aware pack/unpack and MPI
129: .   `PETSCSF_BACKEND_HIP`     - use HIP-aware pack/unpack and MPI
130: -   `PETSCSF_BACKEND_KOKKOS`  - use the Kokkos backend (which itself may dispatch to CUDA, HIP, or OpenMP)

132:    Level: developer

134: .seealso: [](sec_petscsf), `PetscSF`, `PetscSFLink`, `PetscSFDirection`, `PetscSFOperation`
135: E*/
136: /* When doing device-aware MPI, a backend refers to the SF/device interface */
137: typedef enum {
138:   PETSCSF_BACKEND_INVALID = 0,
139:   PETSCSF_BACKEND_CUDA    = 1,
140:   PETSCSF_BACKEND_HIP     = 2,
141:   PETSCSF_BACKEND_KOKKOS  = 3
142: } PetscSFBackend;
143: /*S
144:   PetscSFLink - Opaque internal scratch object used by `PetscSF` to pair a packed buffer with the appropriate pack/unpack and MPI operations for a given root-data layout and `PetscSFBackend`

146:   Level: developer

148: .seealso: `PetscSF`, `PetscSFBackend`, `PetscSFBcastBegin()`, `PetscSFReduceBegin()`
149: S*/
150: typedef struct _n_PetscSFLink *PetscSFLink;

152: /*S
153:   VecScatter - Object used to manage communication of data
154:   between vectors in parallel or between parallel and sequential vectors. Manages both scatters and gathers

156:   Level: beginner

158:   Note:
159:   This is an alias for `PetscSF`.

161: .seealso: [](sec_petscsf), `Vec`, `PetscSF`, `VecScatterCreate()`, `VecScatterBegin()`, `VecScatterEnd()`
162: S*/
163: typedef PetscSF VecScatter;

165: /*J
166:   VecScatterType - String with the name of a PETSc vector scatter type

168:   Level: beginner

170:   Note:
171:   This is an alias for `PetscSFType`

173: .seealso: [](sec_petscsf), `PetscSFType`, `VecScatterSetType()`, `VecScatter`, `VecScatterCreate()`, `VecScatterDestroy()`
174: J*/
175: typedef PetscSFType VecScatterType;