Actual source code: petscdevice.h
1: #if !defined(PETSCDEVICE_H)
2: #define PETSCDEVICE_H
4: #include <petscsys.h>
5: #include <petscdevicetypes.h>
6: #include <petscpkg_version.h>
8: #if defined(PETSC_HAVE_CUDA)
9: #include <cuda.h>
10: #include <cuda_runtime.h>
11: #include <cublas_v2.h>
12: #include <cusolverDn.h>
13: #include <cusolverSp.h>
14: #include <cufft.h>
16: /* cuBLAS does not have cublasGetErrorName(). We create one on our own. */
17: PETSC_EXTERN const char* PetscCUBLASGetErrorName(cublasStatus_t); /* PETSC_EXTERN since it is exposed by the CHKERRCUBLAS macro */
18: PETSC_EXTERN const char* PetscCUSolverGetErrorName(cusolverStatus_t);
19: PETSC_EXTERN const char* PetscCUFFTGetErrorName(cufftResult);
21: /* REMOVE ME */
22: #define WaitForCUDA() cudaDeviceSynchronize()
24: /* CUDART_VERSION = 1000 x major + 10 x minor version */
26: /* Could not find exactly which CUDART_VERSION introduced cudaGetErrorName. At least it was in CUDA 8.0 (Sep. 2016) */
27: #if PETSC_PKG_CUDA_VERSION_GE(8,0,0)
28: #define PetscCallCUDA(...) do { \
29: const cudaError_t _p_cuda_err__ = __VA_ARGS__; \
30: if (PetscUnlikely(_p_cuda_err__ != cudaSuccess)) { \
31: const char *name = cudaGetErrorName(_p_cuda_err__); \
32: const char *descr = cudaGetErrorString(_p_cuda_err__); \
33: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuda error %d (%s) : %s", \
34: (PetscErrorCode)_p_cuda_err__,name,descr); \
35: } \
36: } while (0)
37: #else /* PETSC_PKG_CUDA_VERSION_GE(8,0,0) */
38: #define PetscCallCUDA(...) do { \
39: const cudaError_t _p_cuda_err__ = __VA_ARGS__; \
41: } while (0)
42: #endif /* PETSC_PKG_CUDA_VERSION_GE(8,0,0) */
43: #define CHKERRCUDA(...) PetscCallCUDA(__VA_ARGS__)
45: #define PetscCallCUBLAS(...) do { \
46: const cublasStatus_t _p_cublas_stat__ = __VA_ARGS__; \
47: if (PetscUnlikely(_p_cublas_stat__ != CUBLAS_STATUS_SUCCESS)) { \
48: const char *name = PetscCUBLASGetErrorName(_p_cublas_stat__); \
49: if (((_p_cublas_stat__ == CUBLAS_STATUS_NOT_INITIALIZED) || \
50: (_p_cublas_stat__ == CUBLAS_STATUS_ALLOC_FAILED)) && \
51: PetscDeviceInitialized(PETSC_DEVICE_CUDA)) { \
52: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE, \
53: "cuBLAS error %d (%s). " \
54: "Reports not initialized or alloc failed; " \
55: "this indicates the GPU may have run out resources", \
56: (PetscErrorCode)_p_cublas_stat__,name); \
57: } else { \
58: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuBLAS error %d (%s)", \
59: (PetscErrorCode)_p_cublas_stat__,name); \
60: } \
61: } \
62: } while (0)
63: #define CHKERRCUBLAS(...) PetscCallCUBLAS(__VA_ARGS__)
65: #if (CUSPARSE_VER_MAJOR > 10 || CUSPARSE_VER_MAJOR == 10 && CUSPARSE_VER_MINOR >= 2) /* According to cuda/10.1.168 on OLCF Summit */
66: #define PetscCallCUSPARSE(...)\
67: do {\
68: const cusparseStatus_t _p_cusparse_stat__ = __VA_ARGS__;\
69: if (PetscUnlikely(_p_cusparse_stat__)) {\
70: const char *name = cusparseGetErrorName(_p_cusparse_stat__);\
71: const char *descr = cusparseGetErrorString(_p_cusparse_stat__);\
73: else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuSPARSE errorcode %d (%s) : %s",(int)_p_cusparse_stat__,name,descr);\
74: }\
75: } while (0)
76: #else /* (CUSPARSE_VER_MAJOR > 10 || CUSPARSE_VER_MAJOR == 10 && CUSPARSE_VER_MINOR >= 2) */
77: #define PetscCallCUSPARSE(...) do { \
78: const cusparseStatus_t _p_cusparse_stat__ = __VA_ARGS__; \
80: } while (0)
81: #endif /* (CUSPARSE_VER_MAJOR > 10 || CUSPARSE_VER_MAJOR == 10 && CUSPARSE_VER_MINOR >= 2) */
82: #define CHKERRCUSPARSE(...) PetscCallCUSPARSE(__VA_ARGS__)
84: #define PetscCallCUSOLVER(...) do { \
85: const cusolverStatus_t _p_cusolver_stat__ = __VA_ARGS__; \
86: if (PetscUnlikely(_p_cusolver_stat__ != CUSOLVER_STATUS_SUCCESS)) { \
87: const char *name = PetscCUSolverGetErrorName(_p_cusolver_stat__); \
88: if (((_p_cusolver_stat__ == CUSOLVER_STATUS_NOT_INITIALIZED) || \
89: (_p_cusolver_stat__ == CUSOLVER_STATUS_ALLOC_FAILED) || \
90: (_p_cusolver_stat__ == CUSOLVER_STATUS_INTERNAL_ERROR)) && \
91: PetscDeviceInitialized(PETSC_DEVICE_CUDA)) { \
92: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE, \
93: "cuSolver error %d (%s). " \
94: "This indicates the GPU may have run out resources", \
95: (PetscErrorCode)_p_cusolver_stat__,name); \
96: } else { \
97: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuSolver error %d (%s)", \
98: (PetscErrorCode)_p_cusolver_stat__,name); \
99: } \
100: } \
101: } while (0)
102: #define CHKERRCUSOLVER(...) PetscCallCUSOLVER(__VA_ARGS__)
104: #define PetscCallCUFFT(...) do { \
105: const cufftResult_t _p_cufft_stat__ = __VA_ARGS__; \
106: if (PetscUnlikely(_p_cufft_stat__ != CUFFT_SUCCESS)) { \
107: const char *name = PetscCUFFTGetErrorName(_p_cufft_stat__); \
108: if (((_p_cufft_stat__ == CUFFT_SETUP_FAILED) || \
109: (_p_cufft_stat__ == CUFFT_ALLOC_FAILED)) && \
110: PetscDeviceInitialized(PETSC_DEVICE_CUDA)) { \
111: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE, \
112: "cuFFT error %d (%s). " \
113: "Reports not initialized or alloc failed; " \
114: "this indicates the GPU has run out resources", \
115: (PetscErrorCode)_p_cufft_stat__,name); \
116: } else { \
117: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"cuFFT error %d (%s)", \
118: (PetscErrorCode)_p_cufft_stat__,name); \
119: } \
120: } \
121: } while (0)
122: #define CHKERRCUFFT(...) PetscCallCUFFT(__VA_ARGS__)
124: #define PetscCallCURAND(...) do { \
125: const curandStatus_t _p_curand_stat__ = __VA_ARGS__; \
126: if (PetscUnlikely(_p_curand_stat__ != CURAND_STATUS_SUCCESS)) { \
127: if (((_p_curand_stat__ == CURAND_STATUS_INITIALIZATION_FAILED) || \
128: (_p_curand_stat__ == CURAND_STATUS_ALLOCATION_FAILED)) && \
129: PetscDeviceInitialized(PETSC_DEVICE_CUDA)) { \
130: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU_RESOURCE, \
131: "cuRAND error %d. " \
132: "Reports not initialized or alloc failed; " \
133: "this indicates the GPU has run out resources", \
134: (PetscErrorCode)_p_curand_stat__); \
135: } else { \
136: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU, \
137: "cuRand error %d",(PetscErrorCode)_p_curand_stat__); \
138: } \
139: } \
140: } while (0)
141: #define CHKERRCURAND(...) PetscCallCURAND(__VA_ARGS__)
143: PETSC_EXTERN cudaStream_t PetscDefaultCudaStream; /* The default stream used by PETSc */
145: PETSC_EXTERN PetscErrorCode PetscCUBLASGetHandle(cublasHandle_t*);
146: PETSC_EXTERN PetscErrorCode PetscCUSOLVERDnGetHandle(cusolverDnHandle_t*);
147: #endif /* PETSC_HAVE_CUDA */
149: #if defined(PETSC_HAVE_HIP)
150: #include <hip/hip_runtime.h>
151: #include <hipblas.h>
152: #if defined(__HIP_PLATFORM_NVCC__)
153: #include <cusolverDn.h>
154: #else /* __HIP_PLATFORM_HCC__ */
155: #include <rocsolver.h>
156: #endif /* __HIP_PLATFORM_NVCC__ */
158: /* REMOVE ME */
159: #define WaitForHIP() hipDeviceSynchronize()
161: /* hipBLAS does not have hipblasGetErrorName(). We create one on our own. */
162: PETSC_EXTERN const char* PetscHIPBLASGetErrorName(hipblasStatus_t); /* PETSC_EXTERN since it is exposed by the CHKERRHIPBLAS macro */
164: #define PetscCallHIP(...) do { \
165: const hipError_t _p_hip_err__ = __VA_ARGS__; \
166: if (PetscUnlikely(_p_hip_err__ != hipSuccess)) { \
167: const char *name = hipGetErrorName(_p_hip_err__); \
168: const char *descr = hipGetErrorString(_p_hip_err__); \
169: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"hip error %d (%s) : %s", \
170: (PetscErrorCode)_p_hip_err__,name,descr); \
171: } \
172: } while (0)
173: #define CHKERRHIP(...) PetscCallHIP(__VA_ARGS__)
175: #define PetscCallHIPBLAS(...) do { \
176: const hipblasStatus_t _p_hipblas_stat__ = __VA_ARGS__; \
177: if (PetscUnlikely(_p_hipblas_stat__ != HIPBLAS_STATUS_SUCCESS)) { \
178: const char *name = PetscHIPBLASGetErrorName(_p_hipblas_stat__); \
179: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_GPU,"hipBLAS error %d (%s)", \
180: (PetscErrorCode)_p_hipblas_stat__,name); \
181: } \
182: } while (0)
183: #define CHKERRHIPBLAS(...) PetscCallHIPBLAS(__VA_ARGS__)
185: /* TODO: SEK: Need to figure out the hipsolver issues */
186: #define PetscCallHIPSOLVER(...) do { \
187: const hipsolverStatus_t _p_hipsolver_stat__ = __VA_ARGS__; \
189: } while (0)
190: #define CHKERRHIPSOLVER(...) PetscCallHIPSOLVER(__VA_ARGS__)
192: /* hipSolver does not exist yet so we work around it
193: rocSOLVER users rocBLAS for the handle
194: * */
195: #if defined(__HIP_PLATFORM_NVCC__)
196: typedef cusolverDnHandle_t hipsolverHandle_t;
197: typedef cusolverStatus_t hipsolverStatus_t;
199: /* Alias hipsolverDestroy to cusolverDnDestroy */
200: static inline hipsolverStatus_t hipsolverDestroy(hipsolverHandle_t *hipsolverhandle)
201: {
202: return cusolverDnDestroy(hipsolverhandle);
203: }
205: /* Alias hipsolverCreate to cusolverDnCreate */
206: static inline hipsolverStatus_t hipsolverCreate(hipsolverHandle_t *hipsolverhandle)
207: {
208: return cusolverDnCreate(hipsolverhandle);
209: }
211: /* Alias hipsolverGetStream to cusolverDnGetStream */
212: static inline hipsolverStatus_t hipsolverGetStream(hipsolverHandle_t handle, hipStream_t *stream)
213: {
214: return cusolverDnGetStream(handle,stream);
215: }
217: /* Alias hipsolverSetStream to cusolverDnSetStream */
218: static inline hipsolverStatus_t hipsolverSetStream(hipsolverHandle_t handle, hipStream_t stream)
219: {
220: return cusolveDnSetStream(handle,stream);
221: }
222: #else /* __HIP_PLATFORM_HCC__ */
223: typedef rocblas_handle hipsolverHandle_t;
224: typedef rocblas_status hipsolverStatus_t;
226: /* Alias hipsolverDestroy to rocblas_destroy_handle */
227: static inline hipsolverStatus_t hipsolverDestroy(hipsolverHandle_t hipsolverhandle)
228: {
229: return rocblas_destroy_handle(hipsolverhandle);
230: }
232: /* Alias hipsolverCreate to rocblas_destroy_handle */
233: static inline hipsolverStatus_t hipsolverCreate(hipsolverHandle_t *hipsolverhandle)
234: {
235: return rocblas_create_handle(hipsolverhandle);
236: }
238: /* Alias hipsolverGetStream to rocblas_get_stream */
239: static inline hipsolverStatus_t hipsolverGetStream(hipsolverHandle_t handle, hipStream_t *stream)
240: {
241: return rocblas_get_stream(handle,stream);
242: }
244: /* Alias hipsolverSetStream to rocblas_set_stream */
245: static inline hipsolverStatus_t hipsolverSetStream(hipsolverHandle_t handle, hipStream_t stream)
246: {
247: return rocblas_set_stream(handle,stream);
248: }
249: #endif /* __HIP_PLATFORM_NVCC__ */
250: PETSC_EXTERN hipStream_t PetscDefaultHipStream; /* The default stream used by PETSc */
252: PETSC_EXTERN PetscErrorCode PetscHIPBLASGetHandle(hipblasHandle_t*);
253: PETSC_EXTERN PetscErrorCode PetscHIPSOLVERGetHandle(hipsolverHandle_t*);
254: #endif /* PETSC_HAVE_HIP */
256: /* Cannot use the device context api without C++ */
257: #if defined(PETSC_HAVE_CXX)
258: PETSC_EXTERN PetscErrorCode PetscDeviceInitializePackage(void);
259: PETSC_EXTERN PetscErrorCode PetscDeviceFinalizePackage(void);
261: /* PetscDevice */
262: PETSC_EXTERN PetscErrorCode PetscDeviceInitialize(PetscDeviceType);
263: PETSC_EXTERN PetscBool PetscDeviceInitialized(PetscDeviceType);
264: PETSC_EXTERN PetscErrorCode PetscDeviceCreate(PetscDeviceType,PetscInt,PetscDevice*);
265: PETSC_EXTERN PetscErrorCode PetscDeviceConfigure(PetscDevice);
266: PETSC_EXTERN PetscErrorCode PetscDeviceView(PetscDevice,PetscViewer);
267: PETSC_EXTERN PetscErrorCode PetscDeviceDestroy(PetscDevice*);
268: PETSC_EXTERN PetscErrorCode PetscDeviceGetDeviceId(PetscDevice,PetscInt*);
270: /* PetscDeviceContext */
271: PETSC_EXTERN PetscErrorCode PetscDeviceContextCreate(PetscDeviceContext*);
272: PETSC_EXTERN PetscErrorCode PetscDeviceContextDestroy(PetscDeviceContext*);
273: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetDevice(PetscDeviceContext,PetscDevice);
274: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetDevice(PetscDeviceContext,PetscDevice*);
275: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetStreamType(PetscDeviceContext,PetscStreamType);
276: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetStreamType(PetscDeviceContext,PetscStreamType*);
277: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetUp(PetscDeviceContext);
278: PETSC_EXTERN PetscErrorCode PetscDeviceContextDuplicate(PetscDeviceContext,PetscDeviceContext*);
279: PETSC_EXTERN PetscErrorCode PetscDeviceContextQueryIdle(PetscDeviceContext,PetscBool*);
280: PETSC_EXTERN PetscErrorCode PetscDeviceContextWaitForContext(PetscDeviceContext,PetscDeviceContext);
281: PETSC_EXTERN PetscErrorCode PetscDeviceContextFork(PetscDeviceContext,PetscInt,PetscDeviceContext**);
282: PETSC_EXTERN PetscErrorCode PetscDeviceContextJoin(PetscDeviceContext,PetscInt,PetscDeviceContextJoinMode,PetscDeviceContext**);
283: PETSC_EXTERN PetscErrorCode PetscDeviceContextSynchronize(PetscDeviceContext);
284: PETSC_EXTERN PetscErrorCode PetscDeviceContextGetCurrentContext(PetscDeviceContext*);
285: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetCurrentContext(PetscDeviceContext);
286: PETSC_EXTERN PetscErrorCode PetscDeviceContextSetFromOptions(MPI_Comm,const char[],PetscDeviceContext);
287: #endif /* PETSC_HAVE_CXX */
289: #endif /* PETSCDEVICE_H */