Actual source code: vecseqcupm.cu
1: #include "../vecseqcupm.hpp" /*I <petscvec.h> I*/
2: #include "../vecseqcupm_impl.hpp"
4: using namespace ::Petsc::vec::cupm;
5: using ::Petsc::device::cupm::DeviceType;
7: template class impl::VecSeq_CUPM<DeviceType::CUDA>;
9: static constexpr auto VecSeq_CUDA = impl::VecSeq_CUPM<DeviceType::CUDA>{};
11: /*MC
12: VECSEQCUDA - VECSEQCUDA = "seqcuda" - The basic sequential vector, modified to use CUDA
14: Options Database Key:
15: . -vec_type seqcuda - sets the vector type to `VECSEQCUDA` during a call to `VecSetFromOptions()`
17: Level: beginner
19: .seealso: `VecCreate()`, `VecSetType()`, `VecSetFromOptions()`, `VecCreateMPIWithArray()`, `VECSEQ`,
20: `VecType`, `VecCreateMPI()`, `VecSetPinnedMemoryMin()`, `VECCUDA`, `VECHIP`, `VECMPICUDA`, `VECMPIHIP`, `VECSEQHIP`
21: M*/
23: PetscErrorCode VecCreate_SeqCUDA(Vec v)
24: {
25: PetscFunctionBegin;
26: PetscCall(VecSeq_CUDA.Create(v));
27: PetscFunctionReturn(PETSC_SUCCESS);
28: }
30: PetscErrorCode VecConvert_Seq_SeqCUDA_inplace(Vec v)
31: {
32: PetscFunctionBegin;
33: PetscCall(VecSeq_CUDA.Convert_IMPL_IMPLCUPM(v));
34: PetscFunctionReturn(PETSC_SUCCESS);
35: }
37: // PetscClangLinter pragma disable: -fdoc-internal-linkage
38: /*@
39: VecCreateSeqCUDA - Creates a standard, sequential, array-style vector.
41: Collective, Possibly Synchronous
43: Input Parameters:
44: + comm - the communicator, must be `PETSC_COMM_SELF`
45: - n - the vector length
47: Output Parameter:
48: . v - the vector
50: Level: intermediate
52: Notes:
53: Use `VecDuplicate()` or `VecDuplicateVecs()` to form additional vectors of the same type as an
54: existing vector.
56: This function may initialize `PetscDevice`, which may incur a device synchronization.
58: .seealso: [](ch_vectors), `Vec`, `VECSEQCUDA`, `PetscDeviceInitialize()`, `VecCreate()`, `VecCreateSeq()`, `VecCreateSeqCUDAWithArray()`,
59: `VecCreateMPI()`, `VecCreateMPICUDA()`, `VecDuplicate()`, `VecDuplicateVecs()`, `VecCreateGhost()`
60: @*/
61: PetscErrorCode VecCreateSeqCUDA(MPI_Comm comm, PetscInt n, Vec *v)
62: {
63: PetscFunctionBegin;
64: PetscCall(VecCreateSeqCUPMAsync<DeviceType::CUDA>(comm, n, v));
65: PetscFunctionReturn(PETSC_SUCCESS);
66: }
68: // PetscClangLinter pragma disable: -fdoc-internal-linkage
69: /*@
70: VecCreateSeqCUDAWithArrays - Creates a sequential, array-style vector using CUDA, where the
71: user provides the complete array space to store the vector values.
73: Collective, Possibly Synchronous
75: Input Parameters:
76: + comm - the communicator, must be `PETSC_COMM_SELF`
77: . bs - the block size
78: . n - the local vector length
79: . cpuarray - CPU memory where the vector elements are to be stored (or `NULL`)
80: - gpuarray - GPU memory where the vector elements are to be stored (or `NULL`)
82: Output Parameter:
83: . v - the vector
85: Level: intermediate
87: Notes:
88: If the user-provided array is `NULL`, then `VecCUDAPlaceArray()` can be used at a later stage to
89: SET the array for storing the vector values. Otherwise, the array must be allocated on the
90: device.
92: If both `cpuarray` and `gpuarray` are provided, the provided arrays must have identical
93: values.
95: The arrays are NOT freed when the vector is destroyed via `VecDestroy()`. The user must free
96: them themselves, but not until the vector is destroyed.
98: This function may initialize `PetscDevice`, which may incur a device synchronization.
100: .seealso: [](ch_vectors), `PetscDeviceInitialize()`, `VecCreate()`, `VecCreateSeqWithArray()`, `VecCreateSeqCUDA()`,
101: `VecCreateSeqCUDAWithArray()`, `VecCreateMPICUDA()`, `VecCreateMPICUDAWithArray()`,
102: `VecCreateMPICUDAWithArrays()`, `VecCUDAPlaceArray()`
103: @*/
104: PetscErrorCode VecCreateSeqCUDAWithArrays(MPI_Comm comm, PetscInt bs, PetscInt n, const PetscScalar cpuarray[], const PetscScalar gpuarray[], Vec *v)
105: {
106: PetscFunctionBegin;
107: PetscCall(VecCreateSeqCUPMWithArraysAsync<DeviceType::CUDA>(comm, bs, n, cpuarray, gpuarray, v));
108: PetscFunctionReturn(PETSC_SUCCESS);
109: }
111: // PetscClangLinter pragma disable: -fdoc-internal-linkage
112: /*@
113: VecCreateSeqCUDAWithArray - Creates a sequential, array-style vector using CUDA, where the
114: user provides the device array space to store the vector values.
116: Collective, Possibly Synchronous
118: Input Parameters:
119: + comm - the communicator, must be `PETSC_COMM_SELF`
120: . bs - the block size
121: . n - the vector length
122: - gpuarray - GPU memory where the vector elements are to be stored (or `NULL`)
124: Output Parameter:
125: . v - the vector
127: Level: intermediate
129: Notes:
130: If the user-provided array is `NULL`, then `VecCUDAPlaceArray()` can be used at a later stage to
131: SET the array for storing the vector values. Otherwise, the array must be allocated on the
132: device.
134: The array is NOT freed when the vector is destroyed via `VecDestroy()`. The user must free the
135: array themselves, but not until the vector is destroyed.
137: Use `VecDuplicate()` or `VecDuplicateVecs()` to form additional vectors of the same type as an
138: existing vector.
140: This function may initialize `PetscDevice`, which may incur a device synchronization.
142: .seealso: [](ch_vectors), `PetscDeviceInitialize()`, `VecCreate()`, `VecCreateSeq()`, `VecCreateSeqWithArray()`, `VecCreateSeqWithArrayAndMemType()`,
143: `VecCreateMPIWithArray()`, `VecCreateSeqCUDA()`, `VecCreateMPICUDAWithArray()`, `VecCUDAPlaceArray()`,
144: `VecDuplicate()`, `VecDuplicateVecs()`, `VecCreateGhost()`
145: @*/
146: PetscErrorCode VecCreateSeqCUDAWithArray(MPI_Comm comm, PetscInt bs, PetscInt n, const PetscScalar gpuarray[], Vec *v)
147: {
148: PetscFunctionBegin;
149: PetscCall(VecCreateSeqCUDAWithArrays(comm, bs, n, nullptr, gpuarray, v));
150: PetscFunctionReturn(PETSC_SUCCESS);
151: }
153: // PetscClangLinter pragma disable: -fdoc-internal-linkage
154: /*@
155: VecCUDAGetArray - Provides access to the device buffer inside a vector
157: Logically Collective; Asynchronous
159: Input Parameter:
160: . v - the vector
162: Output Parameter:
163: . a - the device buffer
165: Level: intermediate
167: Notes:
168: This routine has semantics similar to `VecGetArray()`; the returned buffer points to a
169: consistent view of the vector data. This may involve copying data from the host to the device
170: if the data on the device is out of date. It is also assumed that the returned buffer is
171: immediately modified, marking the host data out of date. This is similar to intent(inout) in
172: Fortran.
174: If the user does require strong memory guarantees, they are encouraged to use
175: `VecCUDAGetArrayRead()` and/or `VecCUDAGetArrayWrite()` instead.
177: The user must call `VecCUDARestoreArray()` when they are finished using the array.
179: Developer Note:
180: If the device memory hasn't been allocated previously it will be allocated as part of this
181: routine.
183: Fortran Note:
184: .vb
185: PetscScalar, pointer :: a(:)
186: .ve
187: `a` addresses device memory; pass it to device code and release it with `VecCUDARestoreArray()`.
189: .seealso: [](ch_vectors), `VecCUDARestoreArray()`, `VecCUDAGetArrayRead()`, `VecCUDAGetArrayWrite()`, `VecGetArray()`,
190: `VecGetArrayRead()`, `VecGetArrayWrite()`
191: @*/
192: PetscErrorCode VecCUDAGetArray(Vec v, PetscScalar *a[])
193: {
194: PetscFunctionBegin;
195: PetscCall(VecCUPMGetArrayAsync<DeviceType::CUDA>(v, a));
196: PetscFunctionReturn(PETSC_SUCCESS);
197: }
199: // PetscClangLinter pragma disable: -fdoc-internal-linkage
200: /*@
201: VecCUDARestoreArray - Restore a device buffer previously acquired with `VecCUDAGetArray()`.
203: Not Collective; Asynchronous
205: Input Parameters:
206: + v - the vector
207: - a - the device buffer
209: Level: intermediate
211: Note:
212: The restored pointer is invalid after this function returns. This function also marks the
213: host data as out of date. Subsequent access to the vector data on the host side via
214: `VecGetArray()` will incur a (synchronous) data transfer.
216: Fortran Note:
217: .vb
218: PetscScalar, pointer :: a(:)
219: .ve
221: .seealso: [](ch_vectors), `VecCUDAGetArray()`, `VecCUDAGetArrayRead()`, `VecCUDAGetArrayWrite()`, `VecGetArray()`,
222: `VecRestoreArray()`, `VecGetArrayRead()`
223: @*/
224: PetscErrorCode VecCUDARestoreArray(Vec v, PetscScalar *a[])
225: {
226: PetscFunctionBegin;
227: PetscCall(VecCUPMRestoreArrayAsync<DeviceType::CUDA>(v, a));
228: PetscFunctionReturn(PETSC_SUCCESS);
229: }
231: // PetscClangLinter pragma disable: -fdoc-internal-linkage
232: /*@
233: VecCUDAGetArrayRead - Provides read access to the CUDA buffer inside a vector.
235: Not Collective; Asynchronous
237: Input Parameter:
238: . v - the vector
240: Output Parameter:
241: . a - the CUDA pointer.
243: Level: intermediate
245: Notes:
246: See `VecCUDAGetArray()` for data movement semantics of this function.
248: This function assumes that the user will not modify the vector data. This is analgogous to
249: intent(in) in Fortran.
251: The device pointer must be restored by calling `VecCUDARestoreArrayRead()`. If the data on the
252: host side was previously up to date it will remain so, i.e. data on both the device and the
253: host is up to date. Accessing data on the host side does not incur a device to host data
254: transfer.
256: Fortran Note:
257: .vb
258: PetscScalar, pointer :: a(:)
259: .ve
260: `a` addresses device memory; pass it to device code and release it with `VecCUDARestoreArrayRead()`.
262: .seealso: [](ch_vectors), `VecCUDARestoreArrayRead()`, `VecCUDAGetArray()`, `VecCUDAGetArrayWrite()`, `VecGetArray()`,
263: `VecGetArrayRead()`
264: @*/
265: PetscErrorCode VecCUDAGetArrayRead(Vec v, const PetscScalar *a[])
266: {
267: PetscFunctionBegin;
268: PetscCall(VecCUPMGetArrayReadAsync<DeviceType::CUDA>(v, a));
269: PetscFunctionReturn(PETSC_SUCCESS);
270: }
272: // PetscClangLinter pragma disable: -fdoc-internal-linkage
273: /*@
274: VecCUDARestoreArrayRead - Restore a CUDA device pointer previously acquired with
275: `VecCUDAGetArrayRead()`.
277: Not Collective; Asynchronous
279: Input Parameters:
280: + v - the vector
281: - a - the CUDA device pointer
283: Level: intermediate
285: Note:
286: This routine does not modify the corresponding array on the host in any way. The pointer is
287: invalid after this function returns.
289: Fortran Note:
290: .vb
291: PetscScalar, pointer :: a(:)
292: .ve
294: .seealso: [](ch_vectors), `VecCUDAGetArrayRead()`, `VecCUDAGetArrayWrite()`, `VecCUDAGetArray()`, `VecGetArray()`,
295: `VecRestoreArray()`, `VecGetArrayRead()`
296: @*/
297: PetscErrorCode VecCUDARestoreArrayRead(Vec v, const PetscScalar *a[])
298: {
299: PetscFunctionBegin;
300: PetscCall(VecCUPMRestoreArrayReadAsync<DeviceType::CUDA>(v, a));
301: PetscFunctionReturn(PETSC_SUCCESS);
302: }
304: // PetscClangLinter pragma disable: -fdoc-internal-linkage
305: /*@
306: VecCUDAGetArrayWrite - Provides write access to the CUDA buffer inside a vector.
308: Logically Collective; Asynchronous
310: Input Parameter:
311: . v - the vector
313: Output Parameter:
314: . a - the CUDA pointer
316: Level: advanced
318: Notes:
319: The data pointed to by the device pointer is uninitialized. The user may not read from this
320: data. Furthermore, the entire array needs to be filled by the user to obtain well-defined
321: behaviour. The device memory will be allocated by this function if it hasn't been allocated
322: previously. This is analogous to intent(out) in Fortran.
324: The device pointer needs to be released with `VecCUDARestoreArrayWrite()`. When the pointer is
325: released the host data of the vector is marked as out of data. Subsequent access of the host
326: data with e.g. VecGetArray() incurs a device to host data transfer.
328: Fortran Note:
329: .vb
330: PetscScalar, pointer :: a(:)
331: .ve
332: `a` addresses device memory; pass it to device code and release it with `VecCUDARestoreArrayWrite()`.
334: .seealso: [](ch_vectors), `VecCUDARestoreArrayWrite()`, `VecCUDAGetArray()`, `VecCUDAGetArrayRead()`,
335: `VecCUDAGetArrayWrite()`, `VecGetArray()`, `VecGetArrayRead()`
336: @*/
337: PetscErrorCode VecCUDAGetArrayWrite(Vec v, PetscScalar *a[])
338: {
339: PetscFunctionBegin;
340: PetscCall(VecCUPMGetArrayWriteAsync<DeviceType::CUDA>(v, a));
341: PetscFunctionReturn(PETSC_SUCCESS);
342: }
344: // PetscClangLinter pragma disable: -fdoc-internal-linkage
345: /*@
346: VecCUDARestoreArrayWrite - Restore a CUDA device pointer previously acquired with
347: `VecCUDAGetArrayWrite()`.
349: Logically Collective; Asynchronous
351: Input Parameters:
352: + v - the vector
353: - a - the CUDA device pointer. This pointer is invalid after `VecCUDARestoreArrayWrite()` returns.
355: Level: intermediate
357: Note:
358: Data on the host will be marked as out of date. Subsequent access of the data on the host
359: side e.g. with `VecGetArray()` will incur a device to host data transfer.
361: Fortran Note:
362: .vb
363: PetscScalar, pointer :: a(:)
364: .ve
366: .seealso: [](ch_vectors), `VecCUDAGetArrayWrite()`, `VecCUDAGetArray()`, `VecCUDAGetArrayRead()`,
367: `VecCUDAGetArrayWrite()`, `VecGetArray()`, `VecRestoreArray()`, `VecGetArrayRead()`
368: @*/
369: PetscErrorCode VecCUDARestoreArrayWrite(Vec v, PetscScalar *a[])
370: {
371: PetscFunctionBegin;
372: PetscCall(VecCUPMRestoreArrayWriteAsync<DeviceType::CUDA>(v, a));
373: PetscFunctionReturn(PETSC_SUCCESS);
374: }
376: // PetscClangLinter pragma disable: -fdoc-internal-linkage
377: /*@
378: VecCUDAPlaceArray - Allows one to replace the GPU array in a vector with a GPU array provided
379: by the user.
381: Logically Collective; Asynchronous; No Fortran Support
383: Input Parameters:
384: + vec - the vector
385: - array - the GPU array
387: Level: advanced
389: Notes:
390: Adding `const` to `array` was an oversight, see notes in `VecPlaceArray()`.
392: This routine is useful to avoid copying an array into a vector, though you can return to the
393: original GPU array with a call to `VecCUDAResetArray()`.
395: It is not possible to use `VecCUDAPlaceArray()` and `VecPlaceArray()` at the same time on the
396: same vector.
398: `vec` does not take ownership of `array` in any way. The user must free `array` themselves
399: but be careful not to do so before the vector has either been destroyed, had its original
400: array restored with `VecCUDAResetArray()` or permanently replaced with
401: `VecCUDAReplaceArray()`.
403: .seealso: [](ch_vectors), `VecPlaceArray()`, `VecGetArray()`, `VecRestoreArray()`, `VecReplaceArray()`,
404: `VecResetArray()`, `VecCUDAResetArray()`, `VecCUDAReplaceArray()`
405: @*/
406: PetscErrorCode VecCUDAPlaceArray(Vec vin, const PetscScalar array[])
407: {
408: PetscFunctionBegin;
409: PetscCall(VecCUPMPlaceArrayAsync<DeviceType::CUDA>(vin, array));
410: PetscFunctionReturn(PETSC_SUCCESS);
411: }
413: // PetscClangLinter pragma disable: -fdoc-internal-linkage
414: /*@
415: VecCUDAReplaceArray - Permanently replace the GPU array in a vector with a GPU array provided
416: by the user.
418: Logically Collective; No Fortran Support
420: Input Parameters:
421: + vec - the vector
422: - array - the GPU array
424: Level: advanced
426: Notes:
427: Adding `const` to `array` was an oversight, see notes in `VecPlaceArray()`.
429: This is useful to avoid copying a GPU array into a vector.
431: This frees the memory associated with the old GPU array. The vector takes ownership of the
432: passed array so it CANNOT be freed by the user. It will be freed when the vector is
433: destroyed.
435: .seealso: [](ch_vectors), `VecGetArray()`, `VecRestoreArray()`, `VecPlaceArray()`, `VecResetArray()`,
436: `VecCUDAResetArray()`, `VecCUDAPlaceArray()`, `VecReplaceArray()`
437: @*/
438: PetscErrorCode VecCUDAReplaceArray(Vec vin, const PetscScalar array[])
439: {
440: PetscFunctionBegin;
441: PetscCall(VecCUPMReplaceArrayAsync<DeviceType::CUDA>(vin, array));
442: PetscFunctionReturn(PETSC_SUCCESS);
443: }
445: // PetscClangLinter pragma disable: -fdoc-internal-linkage
446: /*@
447: VecCUDAResetArray - Resets a vector to use its default memory.
449: Logically Collective; No Fortran Support
451: Input Parameters:
452: . vec - the vector
454: Level: advanced
456: Note:
457: Call this after the use of `VecCUDAPlaceArray()`.
459: .seealso: [](ch_vectors), `VecGetArray()`, `VecRestoreArray()`, `VecReplaceArray()`, `VecPlaceArray()`,
460: `VecResetArray()`, `VecCUDAPlaceArray()`, `VecCUDAReplaceArray()`
461: @*/
462: PetscErrorCode VecCUDAResetArray(Vec vin)
463: {
464: PetscFunctionBegin;
465: PetscCall(VecCUPMResetArrayAsync<DeviceType::CUDA>(vin));
466: PetscFunctionReturn(PETSC_SUCCESS);
467: }