Actual source code: mal.c

  1: /*
  2:     Code that allows a user to dictate what malloc() PETSc uses.
  3: */
  4: #include <petscsys.h>
  5: #include <stdarg.h>
  6: #if defined(PETSC_HAVE_MALLOC_H)
  7: #include <malloc.h>
  8: #endif
  9: #if defined(PETSC_HAVE_MEMKIND)
 10: #include <errno.h>
 11: #include <memkind.h>
 12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
 13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
 14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
 15: #endif
 16: /*
 17:         We want to make sure that all mallocs of double or complex numbers are complex aligned.
 18:     1) on systems with memalign() we call that routine to get an aligned memory location
 19:     2) on systems without memalign() we
 20:        - allocate one sizeof(PetscScalar) extra space
 21:        - we shift the pointer up slightly if needed to get PetscScalar aligned
 22:        - if shifted we store at ptr[-1] the amount of shift (plus a classid)
 23: */
 24: #define SHIFT_CLASSID 456123

 26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result)
 27: {
 29: #if defined(PETSC_HAVE_MEMKIND)
 30:   int            err;
 31: #endif

 33:   if (!mem) {*result = NULL; return 0;}
 34: #if defined(PETSC_HAVE_MEMKIND)
 35:   {
 36:     if (!currentmktype) err = memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
 37:     else err = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
 38:     if (err == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
 39:     if (err == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
 40:     if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
 41:     if (clear) {PetscMemzero(*result,mem);}
 42:   }
 43: #else
 44: #  if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
 45:   if (clear) {
 46:     *result = calloc(1+mem/sizeof(int),sizeof(int));
 47:   } else {
 48:     *result = malloc(mem);
 49:   }
 50:   if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
 51:   if (PetscLogMemory) {PetscMemzero(*result,mem);}

 53: #  elif defined(PETSC_HAVE_MEMALIGN)
 54:   *result = memalign(PETSC_MEMALIGN,mem);
 55:   if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
 56:   if (clear || PetscLogMemory) {
 57:     PetscMemzero(*result,mem);
 58:   }
 59: #  else
 60:   {
 61:     int *ptr,shift;
 62:     /*
 63:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
 64:     */
 65:     if (clear) {
 66:       ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
 67:     } else {
 68:       ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
 69:     }
 70:     if (!ptr) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
 71:     shift        = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
 72:     shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
 73:     ptr[shift-1] = shift + SHIFT_CLASSID;
 74:     ptr         += shift;
 75:     *result      = (void*)ptr;
 76:     if (PetscLogMemory) {PetscMemzero(*result,mem);}
 77:   }
 78: #  endif
 79: #endif
 80:   return 0;
 81: }

 83: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
 84: {
 85:   if (!ptr) return 0;
 86: #if defined(PETSC_HAVE_MEMKIND)
 87:   memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
 88: #else
 89: #  if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
 90:   {
 91:     /*
 92:       Previous int tells us how many ints the pointer has been shifted from
 93:       the original address provided by the system malloc().
 94:     */
 95:     int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
 96:     if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
 97:     if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
 98:     ptr = (void*)(((int*)ptr) - shift);
 99:   }
100: #  endif

102: #  if defined(PETSC_HAVE_FREE_RETURN_INT)
103:   int err = free(ptr);
104:   if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
105: #  else
106:   free(ptr);
107: #  endif
108: #endif
109:   return 0;
110: }

112: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
113: {

116:   if (!mem) {
117:     PetscFreeAlign(*result, line, func, file);
118:     if (ierr) return ierr;
119:     *result = NULL;
120:     return 0;
121:   }
122: #if defined(PETSC_HAVE_MEMKIND)
123:   if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
124:   else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
125: #else
126: #  if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
127:   {
128:     /*
129:       Previous int tells us how many ints the pointer has been shifted from
130:       the original address provided by the system malloc().
131:     */
132:     int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
133:     if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
134:     if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
135:     *result = (void*)(((int*)*result) - shift);
136:   }
137: #  endif

139: #  if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
140:   *result = realloc(*result, mem);
141: #  else
142:   {
143:     /*
144:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
145:     */
146:     int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
147:     if (ptr) {
148:       int shift    = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
149:       shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
150:       ptr[shift-1] = shift + SHIFT_CLASSID;
151:       ptr         += shift;
152:       *result      = (void*)ptr;
153:     } else {
154:       *result      = NULL;
155:     }
156:   }
157: #  endif
158: #endif
159:   if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
160: #if defined(PETSC_HAVE_MEMALIGN)
161:   /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
162:    * realloc and, if the alignment is wrong, malloc/copy/free. */
163:   if (((size_t) (*result)) % PETSC_MEMALIGN) {
164:     void *newResult;
165: #  if defined(PETSC_HAVE_MEMKIND)
166:     {
167:       int err;
168:       if (!currentmktype) err = memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
169:       else err = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
170:       if (err == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
171:       if (err == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
172:     }
173: #  else
174:     newResult = memalign(PETSC_MEMALIGN,mem);
175: #  endif
176:     if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
177:     PetscMemcpy(newResult,*result,mem);
178:     if (ierr) return ierr;
179: #  if defined(PETSC_HAVE_FREE_RETURN_INT)
180:     {
181:       int err = free(*result);
182:       if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
183:     }
184: #  else
185: #    if defined(PETSC_HAVE_MEMKIND)
186:     memkind_free(0,*result);
187: #    else
188:     free(*result);
189: #    endif
190: #  endif
191:     *result = newResult;
192:   }
193: #endif
194:   return 0;
195: }

197: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
198: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[])                     = PetscFreeAlign;
199: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;

201: PETSC_INTERN PetscBool petscsetmallocvisited;
202: PetscBool petscsetmallocvisited = PETSC_FALSE;

204: /*@C
205:    PetscMallocSet - Sets the routines used to do mallocs and frees.
206:    This routine MUST be called before PetscInitialize() and may be
207:    called only once.

209:    Not Collective

211:    Input Parameters:
212: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
213: . ifree - the routine that provides the free
214: - iralloc - the routine that provides the realloc

216:    Level: developer

218: @*/
219: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),
220:                               PetscErrorCode (*ifree)(void*,int,const char[],const char[]),
221:                               PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
222: {
224:   if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
225:   PetscTrMalloc         = imalloc;
226:   PetscTrFree           = ifree;
227:   PetscTrRealloc        = iralloc;
228:   petscsetmallocvisited = PETSC_TRUE;
229:   return(0);
230: }

232: /*@C
233:    PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.

235:    Not Collective

237:    Level: developer

239:    Notes:
240:     In general one should never run a PETSc program with different malloc() and
241:     free() settings for different parts; this is because one NEVER wants to
242:     free() an address that was malloced by a different memory management system

244:     Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information

246: @*/
247: PetscErrorCode PetscMallocClear(void)
248: {
250:   PetscTrMalloc         = PetscMallocAlign;
251:   PetscTrFree           = PetscFreeAlign;
252:   PetscTrRealloc        = PetscReallocAlign;
253:   petscsetmallocvisited = PETSC_FALSE;
254:   return(0);
255: }

257: PetscErrorCode PetscMemoryTrace(const char label[])
258: {
259:   PetscErrorCode        ierr;
260:   PetscLogDouble        mem,mal;
261:   static PetscLogDouble oldmem = 0,oldmal = 0;

264:   PetscMemoryGetCurrentUsage(&mem);
265:   PetscMallocGetCurrentUsage(&mal);

267:   PetscPrintf(PETSC_COMM_WORLD,"%s High water  %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
268:   oldmem = mem;
269:   oldmal = mal;
270:   return(0);
271: }

273: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
274: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;
275: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[])                     = PetscFreeAlign;

277: /*@C
278:    PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
279:      If memkind is available, change the memkind type. Otherwise, switch the
280:      current malloc and free routines to the PetscMallocAlign and
281:      PetscFreeAlign (PETSc default).

283:    Not Collective

285:    Level: developer

287:    Notes:
288:      This provides a way to do the allocation on DRAM temporarily. One
289:      can switch back to the previous choice by calling PetscMallocReset().

291: .seealso: PetscMallocReset()
292: @*/
293: PetscErrorCode PetscMallocSetDRAM(void)
294: {
296:   if (PetscTrMalloc == PetscMallocAlign) {
297: #if defined(PETSC_HAVE_MEMKIND)
298:     previousmktype = currentmktype;
299:     currentmktype  = PETSC_MK_DEFAULT;
300: #endif
301:   } else {
302:     /* Save the previous choice */
303:     PetscTrMallocOld  = PetscTrMalloc;
304:     PetscTrReallocOld = PetscTrRealloc;
305:     PetscTrFreeOld    = PetscTrFree;
306:     PetscTrMalloc     = PetscMallocAlign;
307:     PetscTrFree       = PetscFreeAlign;
308:     PetscTrRealloc    = PetscReallocAlign;
309:   }
310:   return(0);
311: }

313: /*@C
314:    PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM

316:    Not Collective

318:    Level: developer

320: .seealso: PetscMallocSetDRAM()
321: @*/
322: PetscErrorCode PetscMallocResetDRAM(void)
323: {
325:   if (PetscTrMalloc == PetscMallocAlign) {
326: #if defined(PETSC_HAVE_MEMKIND)
327:     currentmktype = previousmktype;
328: #endif
329:   } else {
330:     /* Reset to the previous choice */
331:     PetscTrMalloc  = PetscTrMallocOld;
332:     PetscTrRealloc = PetscTrReallocOld;
333:     PetscTrFree    = PetscTrFreeOld;
334:   }
335:   return(0);
336: }

338: static PetscBool petscmalloccoalesce =
339: #if defined(PETSC_USE_MALLOC_COALESCED)
340:   PETSC_TRUE;
341: #else
342:   PETSC_FALSE;
343: #endif

345: /*@C
346:    PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects

348:    Not Collective

350:    Input Parameters:
351: .  coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.

353:    Options Database Keys:
354: .  -malloc_coalesce - turn coalesced malloc on or off

356:    Note:
357:    PETSc uses coalesced malloc by default for optimized builds and not for debugging builds.  This default can be changed via the command-line option -malloc_coalesce or by calling this function.
358:    This function can only be called immediately after PetscInitialize()

360:    Level: developer

362: .seealso: PetscMallocA()
363: @*/
364: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
365: {
367:   petscmalloccoalesce = coalesce;
368:   return(0);
369: }

371: /*@C
372:    PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc

374:    Not Collective

376:    Input Parameters:
377: +  n - number of objects to allocate (at least 1)
378: .  clear - use calloc() to allocate space initialized to zero
379: .  lineno - line number to attribute allocation (typically __LINE__)
380: .  function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
381: .  filename - file name to attribute allocation (typically __FILE__)
382: -  bytes0 - first of n object sizes

384:    Output Parameters:
385: .  ptr0 - first of n pointers to allocate

387:    Notes:
388:    This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.

390:    Level: developer

392: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
393: @*/
394: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
395: {
397:   va_list        Argp;
398:   size_t         bytes[8],sumbytes;
399:   void           **ptr[8];
400:   int            i;

403:   if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
404:   bytes[0] = bytes0;
405:   ptr[0] = (void**)ptr0;
406:   sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
407:   va_start(Argp,ptr0);
408:   for (i=1; i<n; i++) {
409:     bytes[i] = va_arg(Argp,size_t);
410:     ptr[i] = va_arg(Argp,void**);
411:     sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
412:   }
413:   va_end(Argp);
414:   if (petscmalloccoalesce) {
415:     char *p;
416:     (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
417:     for (i=0; i<n; i++) {
418:       *ptr[i] = bytes[i] ? p : NULL;
419:       p = (char*)PetscAddrAlign(p + bytes[i]);
420:     }
421:   } else {
422:     for (i=0; i<n; i++) {
423:       (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
424:     }
425:   }
426:   return(0);
427: }

429: /*@C
430:    PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc

432:    Not Collective

434:    Input Parameters:
435: +  n - number of objects to free (at least 1)
436: .  lineno - line number to attribute deallocation (typically __LINE__)
437: .  function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
438: .  filename - file name to attribute deallocation (typically __FILE__)
439: -  ptr0 ... - first of n pointers to free

441:    Note:
442:    This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.

444:    The pointers are zeroed to prevent users from accidently reusing space that has been freed.

446:    Level: developer

448: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
449: @*/
450: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
451: {
453:   va_list        Argp;
454:   void           **ptr[8];
455:   int            i;

458:   if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
459:   ptr[0] = (void**)ptr0;
460:   va_start(Argp,ptr0);
461:   for (i=1; i<n; i++) {
462:     ptr[i] = va_arg(Argp,void**);
463:   }
464:   va_end(Argp);
465:   if (petscmalloccoalesce) {
466:     for (i=0; i<n; i++) {       /* Find first nonempty allocation */
467:       if (*ptr[i]) break;
468:     }
469:     while (--n > i) {
470:       *ptr[n] = NULL;
471:     }
472:     (*PetscTrFree)(*ptr[n],lineno,function,filename);
473:     *ptr[n] = NULL;
474:   } else {
475:     while (--n >= 0) {
476:       (*PetscTrFree)(*ptr[n],lineno,function,filename);
477:       *ptr[n] = NULL;
478:     }
479:   }
480:   return(0);
481: }