1: #ifndef PETSCDEVICETYPES_H
  2: #define PETSCDEVICETYPES_H

  4: #include <petscmacros.h>

  6: /*E
  7:   PetscMemType - Memory type of a pointer

  9:   Developer Note:
 10:   Encoding of the bitmask in binary: xxxxyyyz

 12: $ z = 0                - Host memory
 13: $ z = 1                - Device memory
 14: $ yyy = 000            - CUDA-related memory
 15: $ yyy = 001            - HIP-related memory
 16: $ yyy = 010            - SYCL-related memory
 17: $ xxxxyyy1 = 0000,0001 - CUDA memory
 18: $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory
 19: $ xxxxyyy1 = 0000,0011 - HIP memory
 20: $ xxxxyyy1 = 0000,0101 - SYCL memory

 22:   Other types of memory, e.g., CUDA managed memory, can be added when needed.

 24:   Level: beginner

 26:   Notes:
 27:   PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration

 29: .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin()
 30: E*/
 31: typedef enum {
 32:   PETSC_MEMTYPE_HOST    = 0,
 33:   PETSC_MEMTYPE_DEVICE  = 0x01,
 34:   PETSC_MEMTYPE_CUDA    = 0x01,
 35:   PETSC_MEMTYPE_NVSHMEM = 0x11,
 36:   PETSC_MEMTYPE_HIP     = 0x03,
 37:   PETSC_MEMTYPE_SYCL    = 0x05,
 38: } PetscMemType;
 39: #if PetscDefined(HAVE_CUDA)
 40: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA
 41: #elif PetscDefined(HAVE_HIP)
 42: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP
 43: #elif PetscDefined(HAVE_SYCL)
 44: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL
 45: #else
 46: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST
 47: #endif

 49: #define PetscMemTypeHost(m)    (((m) & 0x1) == PETSC_MEMTYPE_HOST)
 50: #define PetscMemTypeDevice(m)  (((m) & 0x1) == PETSC_MEMTYPE_DEVICE)
 51: #define PetscMemTypeCUDA(m)    (((m) & 0xF) == PETSC_MEMTYPE_CUDA)
 52: #define PetscMemTypeHIP(m)     (((m) & 0xF) == PETSC_MEMTYPE_HIP)
 53: #define PetscMemTypeSYCL(m)    (((m) & 0xF) == PETSC_MEMTYPE_SYCL)
 54: #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM)

 56: #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)")
 57: /*E
 58:   PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data

 60: $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors
 61: $ PETSC_OFFLOAD_GPU         - GPU has valid vector/matrix entries
 62: $ PETSC_OFFLOAD_CPU         - CPU has valid vector/matrix entries
 63: $ PETSC_OFFLOAD_BOTH        - Both GPU and CPU have valid vector/matrix entries and they match
 64: $ PETSC_OFFLOAD_KOKKOS      - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is.

 66:   Level: developer
 67: E*/
 68: typedef enum {
 69:   PETSC_OFFLOAD_UNALLOCATED = 0x0,
 70:   PETSC_OFFLOAD_CPU         = 0x1,
 71:   PETSC_OFFLOAD_GPU         = 0x2,
 72:   PETSC_OFFLOAD_BOTH        = 0x3,
 73:   PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100,
 74:   PETSC_OFFLOAD_KOKKOS      = 0x100
 75: } PetscOffloadMask;

 77: /*E
 78:   PetscDeviceInitType - Initialization strategy for PetscDevice

 80: $ PETSC_DEVICE_INIT_NONE  - PetscDevice is never initialized
 81: $ PETSC_DEVICE_INIT_LAZY  - PetscDevice is initialized on demand
 82: $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible

 84:   Notes:
 85:   PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and
 86:   doing so results in an error. Useful to ensure that no accelerator is used in a program.

 88:   Level: beginner

 90: .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate()
 91: E*/
 92: typedef enum {
 93:   PETSC_DEVICE_INIT_NONE,
 94:   PETSC_DEVICE_INIT_LAZY,
 95:   PETSC_DEVICE_INIT_EAGER
 96: } PetscDeviceInitType;
 97: PETSC_EXTERN const char *const PetscDeviceInitTypes[];

 99: /*E
100:   PetscDeviceType - Kind of accelerator device backend

102: $ PETSC_DEVICE_INVALID - Invalid type, do not use
103: $ PETSC_DEVICE_CUDA    - CUDA enabled GPU
104: $ PETSC_DEVICE_HIP     - ROCM/HIP enabled GPU
105: $ PETSC_DEVICE_SYCL    - SYCL enabled device
106: $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability
107: $ PETSC_DEVICE_MAX     - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use

109:   Notes:
110:   PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID.

112:   Level: beginner

114: .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate()
115: E*/
116: typedef enum {
117:   PETSC_DEVICE_INVALID,
118:   PETSC_DEVICE_CUDA,
119:   PETSC_DEVICE_HIP,
120:   PETSC_DEVICE_SYCL,
121:   PETSC_DEVICE_MAX
122: } PetscDeviceType;
123: PETSC_EXTERN const char *const PetscDeviceTypes[];
124: #if defined(PETSC_HAVE_HIP)
125: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP
126: #elif defined(PETSC_HAVE_CUDA)
127: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA
128: #elif PetscDefined(HAVE_SYCL)
129: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL
130: #else
131: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID
132: #endif

134: /*S
135:   PetscDevice - Handle to an accelerator "device" (usually a GPU)

137:   Notes:
138:   This object is used to house configuration and state of a device, but does not offer any ability to interact with or
139:   drive device computation. This functionality is facilitated instead by the PetscDeviceContext object.

141:   Level: beginner

143: .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice()
144: S*/
145: typedef struct _n_PetscDevice *PetscDevice;

147: /*E
148:   PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL"
149:   stream, which is usually blocking.

151: $ PETSC_STREAM_GLOBAL_BLOCKING    - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations.
152: $ PETSC_STREAM_DEFAULT_BLOCKING   - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream.
153: $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream.
154: $ PETSC_STREAM_MAX                - Always 1 greater than the largest PetscStreamType, do not use

156:   Level: intermediate

158: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType()
159: E*/
160: typedef enum {
161:   PETSC_STREAM_GLOBAL_BLOCKING,
162:   PETSC_STREAM_DEFAULT_BLOCKING,
163:   PETSC_STREAM_GLOBAL_NONBLOCKING,
164:   PETSC_STREAM_MAX
165: } PetscStreamType;
166: PETSC_EXTERN const char *const PetscStreamTypes[];

168: /*E
169:   PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin()

171: $ PETSC_DEVICE_CONTEXT_JOIN_DESTROY - Destroy all incoming sub-contexts after join.
172: $ PETSC_DEVICE_CONTEXT_JOIN_SYNC    - Synchronize incoming sub-contexts after join.
173: $ PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join.

175:   Level: beginner

177: .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin()
178: E*/
179: typedef enum {
180:   PETSC_DEVICE_CONTEXT_JOIN_DESTROY,
181:   PETSC_DEVICE_CONTEXT_JOIN_SYNC,
182:   PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC
183: } PetscDeviceContextJoinMode;
184: PETSC_EXTERN const char *const PetscDeviceContextJoinModes[];

186: /*S
187:   PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute.

189:   Level: beginner

191: .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(),
192: PetscDeviceContextFork(), PetscDeviceContextJoin()
193: S*/
194: typedef struct _n_PetscDeviceContext *PetscDeviceContext;
195: #endif /* PETSCDEVICETYPES_H */