Actual source code: mal.c
1: /*
2: Code that allows a user to dictate what malloc() PETSc uses.
3: */
4: #include <petscsys.h>
5: #include <stdarg.h>
6: #if defined(PETSC_HAVE_MALLOC_H)
7: #include <malloc.h>
8: #endif
9: #if defined(PETSC_HAVE_MEMKIND)
10: #include <errno.h>
11: #include <memkind.h>
12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
15: #endif
16: /*
17: We want to make sure that all mallocs of double or complex numbers are complex aligned.
18: 1) on systems with memalign() we call that routine to get an aligned memory location
19: 2) on systems without memalign() we
20: - allocate one sizeof(PetscScalar) extra space
21: - we shift the pointer up slightly if needed to get PetscScalar aligned
22: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
23: */
24: #define SHIFT_CLASSID 456123
26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result)
27: {
29: #if defined(PETSC_HAVE_MEMKIND)
30: int err;
31: #endif
33: if (!mem) {*result = NULL; return 0;}
34: #if defined(PETSC_HAVE_MEMKIND)
35: {
36: if (!currentmktype) err = memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
37: else err = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
38: if (err == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
39: if (err == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
40: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
41: if (clear) {PetscMemzero(*result,mem);}
42: }
43: #else
44: # if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
45: if (clear) {
46: *result = calloc(1+mem/sizeof(int),sizeof(int));
47: } else {
48: *result = malloc(mem);
49: }
50: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
51: if (PetscLogMemory) {PetscMemzero(*result,mem);}
53: # elif defined(PETSC_HAVE_MEMALIGN)
54: *result = memalign(PETSC_MEMALIGN,mem);
55: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
56: if (clear || PetscLogMemory) {
57: PetscMemzero(*result,mem);
58: }
59: # else
60: {
61: int *ptr,shift;
62: /*
63: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
64: */
65: if (clear) {
66: ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
67: } else {
68: ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
69: }
70: if (!ptr) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
71: shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
72: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
73: ptr[shift-1] = shift + SHIFT_CLASSID;
74: ptr += shift;
75: *result = (void*)ptr;
76: if (PetscLogMemory) {PetscMemzero(*result,mem);}
77: }
78: # endif
79: #endif
80: return 0;
81: }
83: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
84: {
85: if (!ptr) return 0;
86: #if defined(PETSC_HAVE_MEMKIND)
87: memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
88: #else
89: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
90: {
91: /*
92: Previous int tells us how many ints the pointer has been shifted from
93: the original address provided by the system malloc().
94: */
95: int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
96: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
97: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
98: ptr = (void*)(((int*)ptr) - shift);
99: }
100: # endif
102: # if defined(PETSC_HAVE_FREE_RETURN_INT)
103: int err = free(ptr);
104: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
105: # else
106: free(ptr);
107: # endif
108: #endif
109: return 0;
110: }
112: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
113: {
116: if (!mem) {
117: PetscFreeAlign(*result, line, func, file);
118: if (ierr) return ierr;
119: *result = NULL;
120: return 0;
121: }
122: #if defined(PETSC_HAVE_MEMKIND)
123: if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
124: else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
125: #else
126: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
127: {
128: /*
129: Previous int tells us how many ints the pointer has been shifted from
130: the original address provided by the system malloc().
131: */
132: int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
133: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
134: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
135: *result = (void*)(((int*)*result) - shift);
136: }
137: # endif
139: # if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
140: *result = realloc(*result, mem);
141: # else
142: {
143: /*
144: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
145: */
146: int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
147: if (ptr) {
148: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
149: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
150: ptr[shift-1] = shift + SHIFT_CLASSID;
151: ptr += shift;
152: *result = (void*)ptr;
153: } else {
154: *result = NULL;
155: }
156: }
157: # endif
158: #endif
159: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
160: #if defined(PETSC_HAVE_MEMALIGN)
161: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
162: * realloc and, if the alignment is wrong, malloc/copy/free. */
163: if (((size_t) (*result)) % PETSC_MEMALIGN) {
164: void *newResult;
165: # if defined(PETSC_HAVE_MEMKIND)
166: {
167: int err;
168: if (!currentmktype) err = memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
169: else err = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
170: if (err == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
171: if (err == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
172: }
173: # else
174: newResult = memalign(PETSC_MEMALIGN,mem);
175: # endif
176: if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
177: PetscMemcpy(newResult,*result,mem);
178: if (ierr) return ierr;
179: # if defined(PETSC_HAVE_FREE_RETURN_INT)
180: {
181: int err = free(*result);
182: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
183: }
184: # else
185: # if defined(PETSC_HAVE_MEMKIND)
186: memkind_free(0,*result);
187: # else
188: free(*result);
189: # endif
190: # endif
191: *result = newResult;
192: }
193: #endif
194: return 0;
195: }
197: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
198: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign;
199: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
201: PETSC_INTERN PetscBool petscsetmallocvisited;
202: PetscBool petscsetmallocvisited = PETSC_FALSE;
204: /*@C
205: PetscMallocSet - Sets the routines used to do mallocs and frees.
206: This routine MUST be called before PetscInitialize() and may be
207: called only once.
209: Not Collective
211: Input Parameters:
212: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
213: . ifree - the routine that provides the free
214: - iralloc - the routine that provides the realloc
216: Level: developer
218: @*/
219: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),
220: PetscErrorCode (*ifree)(void*,int,const char[],const char[]),
221: PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
222: {
224: if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
225: PetscTrMalloc = imalloc;
226: PetscTrFree = ifree;
227: PetscTrRealloc = iralloc;
228: petscsetmallocvisited = PETSC_TRUE;
229: return(0);
230: }
232: /*@C
233: PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.
235: Not Collective
237: Level: developer
239: Notes:
240: In general one should never run a PETSc program with different malloc() and
241: free() settings for different parts; this is because one NEVER wants to
242: free() an address that was malloced by a different memory management system
244: Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information
246: @*/
247: PetscErrorCode PetscMallocClear(void)
248: {
250: PetscTrMalloc = PetscMallocAlign;
251: PetscTrFree = PetscFreeAlign;
252: PetscTrRealloc = PetscReallocAlign;
253: petscsetmallocvisited = PETSC_FALSE;
254: return(0);
255: }
257: PetscErrorCode PetscMemoryTrace(const char label[])
258: {
259: PetscErrorCode ierr;
260: PetscLogDouble mem,mal;
261: static PetscLogDouble oldmem = 0,oldmal = 0;
264: PetscMemoryGetCurrentUsage(&mem);
265: PetscMallocGetCurrentUsage(&mal);
267: PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
268: oldmem = mem;
269: oldmal = mal;
270: return(0);
271: }
273: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
274: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
275: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign;
277: /*@C
278: PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
279: If memkind is available, change the memkind type. Otherwise, switch the
280: current malloc and free routines to the PetscMallocAlign and
281: PetscFreeAlign (PETSc default).
283: Not Collective
285: Level: developer
287: Notes:
288: This provides a way to do the allocation on DRAM temporarily. One
289: can switch back to the previous choice by calling PetscMallocReset().
291: .seealso: PetscMallocReset()
292: @*/
293: PetscErrorCode PetscMallocSetDRAM(void)
294: {
296: if (PetscTrMalloc == PetscMallocAlign) {
297: #if defined(PETSC_HAVE_MEMKIND)
298: previousmktype = currentmktype;
299: currentmktype = PETSC_MK_DEFAULT;
300: #endif
301: } else {
302: /* Save the previous choice */
303: PetscTrMallocOld = PetscTrMalloc;
304: PetscTrReallocOld = PetscTrRealloc;
305: PetscTrFreeOld = PetscTrFree;
306: PetscTrMalloc = PetscMallocAlign;
307: PetscTrFree = PetscFreeAlign;
308: PetscTrRealloc = PetscReallocAlign;
309: }
310: return(0);
311: }
313: /*@C
314: PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM
316: Not Collective
318: Level: developer
320: .seealso: PetscMallocSetDRAM()
321: @*/
322: PetscErrorCode PetscMallocResetDRAM(void)
323: {
325: if (PetscTrMalloc == PetscMallocAlign) {
326: #if defined(PETSC_HAVE_MEMKIND)
327: currentmktype = previousmktype;
328: #endif
329: } else {
330: /* Reset to the previous choice */
331: PetscTrMalloc = PetscTrMallocOld;
332: PetscTrRealloc = PetscTrReallocOld;
333: PetscTrFree = PetscTrFreeOld;
334: }
335: return(0);
336: }
338: static PetscBool petscmalloccoalesce =
339: #if defined(PETSC_USE_MALLOC_COALESCED)
340: PETSC_TRUE;
341: #else
342: PETSC_FALSE;
343: #endif
345: /*@C
346: PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects
348: Not Collective
350: Input Parameters:
351: . coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.
353: Options Database Keys:
354: . -malloc_coalesce - turn coalesced malloc on or off
356: Note:
357: PETSc uses coalesced malloc by default for optimized builds and not for debugging builds. This default can be changed via the command-line option -malloc_coalesce or by calling this function.
358: This function can only be called immediately after PetscInitialize()
360: Level: developer
362: .seealso: PetscMallocA()
363: @*/
364: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
365: {
367: petscmalloccoalesce = coalesce;
368: return(0);
369: }
371: /*@C
372: PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc
374: Not Collective
376: Input Parameters:
377: + n - number of objects to allocate (at least 1)
378: . clear - use calloc() to allocate space initialized to zero
379: . lineno - line number to attribute allocation (typically __LINE__)
380: . function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
381: . filename - file name to attribute allocation (typically __FILE__)
382: - bytes0 - first of n object sizes
384: Output Parameters:
385: . ptr0 - first of n pointers to allocate
387: Notes:
388: This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.
390: Level: developer
392: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
393: @*/
394: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
395: {
397: va_list Argp;
398: size_t bytes[8],sumbytes;
399: void **ptr[8];
400: int i;
403: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
404: bytes[0] = bytes0;
405: ptr[0] = (void**)ptr0;
406: sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
407: va_start(Argp,ptr0);
408: for (i=1; i<n; i++) {
409: bytes[i] = va_arg(Argp,size_t);
410: ptr[i] = va_arg(Argp,void**);
411: sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
412: }
413: va_end(Argp);
414: if (petscmalloccoalesce) {
415: char *p;
416: (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
417: for (i=0; i<n; i++) {
418: *ptr[i] = bytes[i] ? p : NULL;
419: p = (char*)PetscAddrAlign(p + bytes[i]);
420: }
421: } else {
422: for (i=0; i<n; i++) {
423: (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
424: }
425: }
426: return(0);
427: }
429: /*@C
430: PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc
432: Not Collective
434: Input Parameters:
435: + n - number of objects to free (at least 1)
436: . lineno - line number to attribute deallocation (typically __LINE__)
437: . function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
438: . filename - file name to attribute deallocation (typically __FILE__)
439: - ptr0 ... - first of n pointers to free
441: Note:
442: This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.
444: The pointers are zeroed to prevent users from accidentally reusing space that has been freed.
446: Level: developer
448: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
449: @*/
450: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
451: {
453: va_list Argp;
454: void **ptr[8];
455: int i;
458: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
459: ptr[0] = (void**)ptr0;
460: va_start(Argp,ptr0);
461: for (i=1; i<n; i++) {
462: ptr[i] = va_arg(Argp,void**);
463: }
464: va_end(Argp);
465: if (petscmalloccoalesce) {
466: for (i=0; i<n; i++) { /* Find first nonempty allocation */
467: if (*ptr[i]) break;
468: }
469: while (--n > i) {
470: *ptr[n] = NULL;
471: }
472: (*PetscTrFree)(*ptr[n],lineno,function,filename);
473: *ptr[n] = NULL;
474: } else {
475: while (--n >= 0) {
476: (*PetscTrFree)(*ptr[n],lineno,function,filename);
477: *ptr[n] = NULL;
478: }
479: }
480: return(0);
481: }