Actual source code: veccuda.c

petsc-3.13.6 2020-09-29
Report Typos and Errors
  1: /*
  2:  Implementation of the sequential cuda vectors.

  4:  This file contains the code that can be compiled with a C
  5:  compiler.  The companion file veccuda2.cu contains the code that
  6:  must be compiled with nvcc or a C++ compiler.
  7:  */

  9: #define PETSC_SKIP_SPINLOCK

 11: #include <petscconf.h>
 12:  #include <petsc/private/vecimpl.h>
 13:  #include <../src/vec/vec/impls/dvecimpl.h>
 14:  #include <petsc/private/cudavecimpl.h>

 16: /*
 17:     Allocates space for the vector array on the Host if it does not exist.
 18:     Does NOT change the PetscCUDAFlag for the vector
 19:     Does NOT zero the CUDA array
 20:  */
 21: PetscErrorCode VecCUDAAllocateCheckHost(Vec v)
 22: {
 24:   PetscScalar    *array;
 25:   Vec_Seq        *s = (Vec_Seq*)v->data;
 26:   PetscInt       n = v->map->n;

 29:   if (!s) {
 30:     PetscNewLog((PetscObject)v,&s);
 31:     v->data = s;
 32:   }
 33:   if (!s->array) {
 34:     if (n*sizeof(PetscScalar) > v->minimum_bytes_pinned_memory) {
 35:       PetscMallocSetCUDAHost();
 36:       v->pinned_memory = PETSC_TRUE;
 37:     }
 38:     PetscMalloc1(n,&array);
 39:     PetscLogObjectMemory((PetscObject)v,n*sizeof(PetscScalar));
 40:     s->array           = array;
 41:     s->array_allocated = array;
 42:     if (n*sizeof(PetscScalar) > v->minimum_bytes_pinned_memory) {
 43:       PetscMallocResetCUDAHost();
 44:     }
 45:     if (v->offloadmask == PETSC_OFFLOAD_UNALLOCATED) {
 46:       v->offloadmask = PETSC_OFFLOAD_CPU;
 47:     }
 48:   }
 49:   return(0);
 50: }

 52: PetscErrorCode VecCopy_SeqCUDA_Private(Vec xin,Vec yin)
 53: {
 54:   PetscScalar       *ya;
 55:   const PetscScalar *xa;
 56:   PetscErrorCode    ierr;

 59:   VecCUDAAllocateCheckHost(xin);
 60:   VecCUDAAllocateCheckHost(yin);
 61:   if (xin != yin) {
 62:     VecGetArrayRead(xin,&xa);
 63:     VecGetArray(yin,&ya);
 64:     PetscArraycpy(ya,xa,xin->map->n);
 65:     VecRestoreArrayRead(xin,&xa);
 66:     VecRestoreArray(yin,&ya);
 67:   }
 68:   return(0);
 69: }

 71: PetscErrorCode VecSetRandom_SeqCUDA_Private(Vec xin,PetscRandom r)
 72: {
 74:   PetscInt       n = xin->map->n,i;
 75:   PetscScalar    *xx;

 78:   VecGetArray(xin,&xx);
 79:   for (i=0; i<n; i++) { PetscRandomGetValue(r,&xx[i]); }
 80:   VecRestoreArray(xin,&xx);
 81:   return(0);
 82: }

 84: PetscErrorCode VecDestroy_SeqCUDA_Private(Vec v)
 85: {
 86:   Vec_Seq        *vs = (Vec_Seq*)v->data;

 90:   PetscObjectSAWsViewOff(v);
 91: #if defined(PETSC_USE_LOG)
 92:   PetscLogObjectState((PetscObject)v,"Length=%D",v->map->n);
 93: #endif
 94:   if (vs) {
 95:     if (vs->array_allocated) {
 96:       if (v->pinned_memory) {
 97:         PetscMallocSetCUDAHost();
 98:       }
 99:       PetscFree(vs->array_allocated);
100:       if (v->pinned_memory) {
101:         PetscMallocResetCUDAHost();
102:         v->pinned_memory = PETSC_FALSE;
103:       }
104:     }
105:     PetscFree(vs);
106:   }
107:   return(0);
108: }

110: PetscErrorCode VecResetArray_SeqCUDA_Private(Vec vin)
111: {
112:   Vec_Seq *v = (Vec_Seq*)vin->data;

115:   v->array         = v->unplacedarray;
116:   v->unplacedarray = 0;
117:   return(0);
118: }

120: PetscErrorCode VecCUDAAllocateCheck_Public(Vec v)
121: {

125:   VecCUDAAllocateCheck(v);
126:   return(0);
127: }

129: PetscErrorCode VecCUDACopyToGPU_Public(Vec v)
130: {

134:   VecCUDACopyToGPU(v);
135:   return(0);
136: }

138: /*
139:     VecCUDACopyToGPUSome_Public - Copies certain entries down to the GPU from the CPU of a vector

141:    Input Parameters:
142:  +  v    - the vector
143:  .  ci   - the requested indices, this should be created with CUDAIndicesCreate()
144:  -  mode - vec scatter mode used in VecScatterBegin/End
145: */
146: PetscErrorCode VecCUDACopyToGPUSome_Public(Vec v,PetscCUDAIndices ci,ScatterMode mode)
147: {

151:   VecCUDACopyToGPUSome(v,ci,mode);
152:   return(0);
153: }

155: /*
156:   VecCUDACopyFromGPUSome_Public - Copies certain entries up to the CPU from the GPU of a vector

158:   Input Parameters:
159:  +  v    - the vector
160:  .  ci   - the requested indices, this should be created with CUDAIndicesCreate()
161:  -  mode - vec scatter mode used in VecScatterBegin/End
162: */
163: PetscErrorCode VecCUDACopyFromGPUSome_Public(Vec v,PetscCUDAIndices ci,ScatterMode mode)
164: {

168:   VecCUDACopyFromGPUSome(v,ci,mode);
169:   return(0);
170: }

172: PetscErrorCode VecSetRandom_SeqCUDA(Vec xin,PetscRandom r)
173: {

177:   VecSetRandom_SeqCUDA_Private(xin,r);
178:   xin->offloadmask = PETSC_OFFLOAD_CPU;
179:   return(0);
180: }

182: PetscErrorCode VecResetArray_SeqCUDA(Vec vin)
183: {

187:   VecCUDACopyFromGPU(vin);
188:   VecResetArray_SeqCUDA_Private(vin);
189:   vin->offloadmask = PETSC_OFFLOAD_CPU;
190:   return(0);
191: }

193: PetscErrorCode VecPlaceArray_SeqCUDA(Vec vin,const PetscScalar *a)
194: {

198:   VecCUDACopyFromGPU(vin);
199:   VecPlaceArray_Seq(vin,a);
200:   vin->offloadmask = PETSC_OFFLOAD_CPU;
201:   return(0);
202: }

204: PetscErrorCode VecReplaceArray_SeqCUDA(Vec vin,const PetscScalar *a)
205: {

209:   VecCUDACopyFromGPU(vin);
210:   VecReplaceArray_Seq(vin,a);
211:   vin->offloadmask = PETSC_OFFLOAD_CPU;
212:   return(0);
213: }

215: /*@
216:  VecCreateSeqCUDA - Creates a standard, sequential array-style vector.

218:  Collective

220:  Input Parameter:
221:  +  comm - the communicator, should be PETSC_COMM_SELF
222:  -  n - the vector length

224:  Output Parameter:
225:  .  v - the vector

227:  Notes:
228:  Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the
229:  same type as an existing vector.

231:  Level: intermediate

233:  .seealso: VecCreateMPI(), VecCreate(), VecDuplicate(), VecDuplicateVecs(), VecCreateGhost()
234:  @*/
235: PetscErrorCode VecCreateSeqCUDA(MPI_Comm comm,PetscInt n,Vec *v)
236: {

240:   VecCreate(comm,v);
241:   VecSetSizes(*v,n,n);
242:   VecSetType(*v,VECSEQCUDA);
243:   return(0);
244: }

246: PetscErrorCode VecDuplicate_SeqCUDA(Vec win,Vec *V)
247: {

251:   VecCreateSeqCUDA(PetscObjectComm((PetscObject)win),win->map->n,V);
252:   PetscLayoutReference(win->map,&(*V)->map);
253:   PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)(*V))->olist);
254:   PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)(*V))->qlist);
255:   (*V)->stash.ignorenegidx = win->stash.ignorenegidx;
256:   return(0);
257: }

259: PetscErrorCode VecCreate_SeqCUDA(Vec V)
260: {

264:   PetscLayoutSetUp(V->map);
265:   VecCUDAAllocateCheck(V);
266:   VecCreate_SeqCUDA_Private(V,((Vec_CUDA*)V->spptr)->GPUarray_allocated);
267:   VecCUDAAllocateCheckHost(V);
268:   VecSet(V,0.0);
269:   VecSet_Seq(V,0.0);
270:   V->offloadmask = PETSC_OFFLOAD_BOTH;
271:   return(0);
272: }

274: /*@C
275:    VecCreateSeqCUDAWithArray - Creates a CUDA sequential array-style vector,
276:    where the user provides the array space to store the vector values. The array
277:    provided must be a GPU array.

279:    Collective

281:    Input Parameter:
282: +  comm - the communicator, should be PETSC_COMM_SELF
283: .  bs - the block size
284: .  n - the vector length
285: -  array - GPU memory where the vector elements are to be stored.

287:    Output Parameter:
288: .  V - the vector

290:    Notes:
291:    Use VecDuplicate() or VecDuplicateVecs() to form additional vectors of the
292:    same type as an existing vector.

294:    If the user-provided array is NULL, then VecCUDAPlaceArray() can be used
295:    at a later stage to SET the array for storing the vector values.

297:    PETSc does NOT free the array when the vector is destroyed via VecDestroy().
298:    The user should not free the array until the vector is destroyed.

300:    Level: intermediate

302: .seealso: VecCreateMPICUDAWithArray(), VecCreate(), VecDuplicate(), VecDuplicateVecs(),
303:           VecCreateGhost(), VecCreateSeq(), VecCUDAPlaceArray(), VecCreateSeqWithArray(),
304:           VecCreateMPIWithArray()
305: @*/
306: PetscErrorCode  VecCreateSeqCUDAWithArray(MPI_Comm comm,PetscInt bs,PetscInt n,const PetscScalar array[],Vec *V)
307: {
309:   PetscMPIInt    size;

312:   VecCreate(comm,V);
313:   VecSetSizes(*V,n,n);
314:   VecSetBlockSize(*V,bs);
315:   MPI_Comm_size(comm,&size);
316:   if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Cannot create VECSEQ on more than one process");
317:   VecCreate_SeqCUDA_Private(*V,array);
318:   (*V)->offloadmask = PETSC_OFFLOAD_GPU;
319:   return(0);
320: }

322: PetscErrorCode VecGetArrayWrite_SeqCUDA(Vec v,PetscScalar **vv)
323: {

327:   VecCUDAAllocateCheckHost(v);
328:   v->offloadmask = PETSC_OFFLOAD_CPU;
329:   *vv = *((PetscScalar**)v->data);
330:   return(0);
331: }

333: PetscErrorCode VecBindToCPU_SeqCUDA(Vec V,PetscBool pin)
334: {

338:   V->boundtocpu = pin;
339:   if (pin) {
340:     VecCUDACopyFromGPU(V);
341:     V->offloadmask                 = PETSC_OFFLOAD_CPU; /* since the CPU code will likely change values in the vector */
342:     V->ops->dot                    = VecDot_Seq;
343:     V->ops->norm                   = VecNorm_Seq;
344:     V->ops->tdot                   = VecTDot_Seq;
345:     V->ops->scale                  = VecScale_Seq;
346:     V->ops->copy                   = VecCopy_Seq;
347:     V->ops->set                    = VecSet_Seq;
348:     V->ops->swap                   = VecSwap_Seq;
349:     V->ops->axpy                   = VecAXPY_Seq;
350:     V->ops->axpby                  = VecAXPBY_Seq;
351:     V->ops->axpbypcz               = VecAXPBYPCZ_Seq;
352:     V->ops->pointwisemult          = VecPointwiseMult_Seq;
353:     V->ops->pointwisedivide        = VecPointwiseDivide_Seq;
354:     V->ops->setrandom              = VecSetRandom_Seq;
355:     V->ops->dot_local              = VecDot_Seq;
356:     V->ops->tdot_local             = VecTDot_Seq;
357:     V->ops->norm_local             = VecNorm_Seq;
358:     V->ops->mdot_local             = VecMDot_Seq;
359:     V->ops->mtdot_local            = VecMTDot_Seq;
360:     V->ops->maxpy                  = VecMAXPY_Seq;
361:     V->ops->mdot                   = VecMDot_Seq;
362:     V->ops->mtdot                  = VecMTDot_Seq;
363:     V->ops->aypx                   = VecAYPX_Seq;
364:     V->ops->waxpy                  = VecWAXPY_Seq;
365:     V->ops->dotnorm2               = NULL;
366:     V->ops->placearray             = VecPlaceArray_Seq;
367:     V->ops->replacearray           = VecReplaceArray_Seq;
368:     V->ops->resetarray             = VecResetArray_Seq;
369:     V->ops->duplicate              = VecDuplicate_Seq;
370:     V->ops->conjugate              = VecConjugate_Seq;
371:     V->ops->getlocalvector         = NULL;
372:     V->ops->restorelocalvector     = NULL;
373:     V->ops->getlocalvectorread     = NULL;
374:     V->ops->restorelocalvectorread = NULL;
375:     V->ops->getarraywrite          = NULL;
376:   } else {
377:     V->ops->dot                    = VecDot_SeqCUDA;
378:     V->ops->norm                   = VecNorm_SeqCUDA;
379:     V->ops->tdot                   = VecTDot_SeqCUDA;
380:     V->ops->scale                  = VecScale_SeqCUDA;
381:     V->ops->copy                   = VecCopy_SeqCUDA;
382:     V->ops->set                    = VecSet_SeqCUDA;
383:     V->ops->swap                   = VecSwap_SeqCUDA;
384:     V->ops->axpy                   = VecAXPY_SeqCUDA;
385:     V->ops->axpby                  = VecAXPBY_SeqCUDA;
386:     V->ops->axpbypcz               = VecAXPBYPCZ_SeqCUDA;
387:     V->ops->pointwisemult          = VecPointwiseMult_SeqCUDA;
388:     V->ops->pointwisedivide        = VecPointwiseDivide_SeqCUDA;
389:     V->ops->setrandom              = VecSetRandom_SeqCUDA;
390:     V->ops->dot_local              = VecDot_SeqCUDA;
391:     V->ops->tdot_local             = VecTDot_SeqCUDA;
392:     V->ops->norm_local             = VecNorm_SeqCUDA;
393:     V->ops->mdot_local             = VecMDot_SeqCUDA;
394:     V->ops->maxpy                  = VecMAXPY_SeqCUDA;
395:     V->ops->mdot                   = VecMDot_SeqCUDA;
396:     V->ops->aypx                   = VecAYPX_SeqCUDA;
397:     V->ops->waxpy                  = VecWAXPY_SeqCUDA;
398:     V->ops->dotnorm2               = VecDotNorm2_SeqCUDA;
399:     V->ops->placearray             = VecPlaceArray_SeqCUDA;
400:     V->ops->replacearray           = VecReplaceArray_SeqCUDA;
401:     V->ops->resetarray             = VecResetArray_SeqCUDA;
402:     V->ops->destroy                = VecDestroy_SeqCUDA;
403:     V->ops->duplicate              = VecDuplicate_SeqCUDA;
404:     V->ops->conjugate              = VecConjugate_SeqCUDA;
405:     V->ops->getlocalvector         = VecGetLocalVector_SeqCUDA;
406:     V->ops->restorelocalvector     = VecRestoreLocalVector_SeqCUDA;
407:     V->ops->getlocalvectorread     = VecGetLocalVector_SeqCUDA;
408:     V->ops->restorelocalvectorread = VecRestoreLocalVector_SeqCUDA;
409:     V->ops->getarraywrite          = VecGetArrayWrite_SeqCUDA;
410:   }
411:   return(0);
412: }

414: PetscErrorCode VecCreate_SeqCUDA_Private(Vec V,const PetscScalar *array)
415: {
417:   Vec_CUDA       *veccuda;
418:   PetscMPIInt    size;
419:   PetscBool      option_set;

422:   MPI_Comm_size(PetscObjectComm((PetscObject)V),&size);
423:   if (size > 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Cannot create VECSEQCUDA on more than one process");
424:   VecCreate_Seq_Private(V,0);
425:   PetscObjectChangeTypeName((PetscObject)V,VECSEQCUDA);
426:   VecBindToCPU_SeqCUDA(V,PETSC_FALSE);
427:   V->ops->bindtocpu = VecBindToCPU_SeqCUDA;

429:   /* Later, functions check for the Vec_CUDA structure existence, so do not create it without array */
430:   if (array) {
431:     if (!V->spptr) {
432:       PetscReal pinned_memory_min;
433:       PetscMalloc(sizeof(Vec_CUDA),&V->spptr);
434:       veccuda = (Vec_CUDA*)V->spptr;
435:       veccuda->stream = 0; /* using default stream */
436:       veccuda->GPUarray_allocated = 0;
437:       V->offloadmask = PETSC_OFFLOAD_UNALLOCATED;

439:       pinned_memory_min = 0;
440:       /* Need to parse command line for minimum size to use for pinned memory allocations on host here.
441:          Note: This same code duplicated in VecCUDAAllocateCheck() and VecCreate_MPICUDA_Private(). Is there a good way to avoid this? */
442:       PetscOptionsBegin(PetscObjectComm((PetscObject)V),((PetscObject)V)->prefix,"VECCUDA Options","Vec");
443:       PetscOptionsReal("-vec_pinned_memory_min","Minimum size (in bytes) for an allocation to use pinned memory on host","VecSetPinnedMemoryMin",pinned_memory_min,&pinned_memory_min,&option_set);
444:       if (option_set) V->minimum_bytes_pinned_memory = pinned_memory_min;
445:       PetscOptionsEnd();
446:     }
447:     veccuda = (Vec_CUDA*)V->spptr;
448:     veccuda->GPUarray = (PetscScalar*)array;
449:   }

451:   return(0);
452: }