Actual source code: mal.c
1: /*
2: Code that allows a user to dictate what malloc() PETSc uses.
3: */
4: #include <petscsys.h>
5: #include <stdarg.h>
6: #if defined(PETSC_HAVE_MALLOC_H)
7: #include <malloc.h>
8: #endif
9: #if defined(PETSC_HAVE_MEMKIND)
10: #include <errno.h>
11: #include <memkind.h>
12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
15: #endif
16: /*
17: We want to make sure that all mallocs of double or complex numbers are complex aligned.
18: 1) on systems with memalign() we call that routine to get an aligned memory location
19: 2) on systems without memalign() we
20: - allocate one sizeof(PetscScalar) extra space
21: - we shift the pointer up slightly if needed to get PetscScalar aligned
22: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
23: */
24: #define SHIFT_CLASSID 456123
26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result)
27: {
28: if (!mem) {*result = NULL; return 0;}
29: #if PetscDefined(HAVE_MEMKIND)
30: {
31: int err;
33: err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
35: if (err == ENOMEM) PetscInfo(NULL,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
37: if (clear) PetscMemzero(*result,mem);
38: }
39: #else /* PetscDefined(HAVE_MEMKIND) */
40: # if PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
41: if (clear) *result = calloc(1+mem/sizeof(int),sizeof(int));
42: else *result = malloc(mem);
45: if (PetscLogMemory) PetscMemzero(*result,mem);
47: # elif PetscDefined(HAVE_MEMALIGN)
48: *result = memalign(PETSC_MEMALIGN,mem);
50: if (clear || PetscLogMemory) PetscMemzero(*result,mem);
51: # else /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_MEMALIGN) */
52: {
53: int *ptr,shift;
54: /*
55: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
56: */
57: if (clear) {
58: ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
59: } else {
60: ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
61: }
63: shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
64: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
65: ptr[shift-1] = shift + SHIFT_CLASSID;
66: ptr += shift;
67: *result = (void*)ptr;
68: if (PetscLogMemory) PetscMemzero(*result,mem);
69: }
70: # endif /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_MEMALIGN) */
71: #endif /* PetscDefined(HAVE_MEMKIND) */
72: return 0;
73: }
75: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
76: {
77: if (!ptr) return 0;
78: #if PetscDefined(HAVE_MEMKIND)
79: memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
80: #else /* PetscDefined(HAVE_MEMKIND) */
81: # if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_MEMALIGN))
82: {
83: /*
84: Previous int tells us how many ints the pointer has been shifted from
85: the original address provided by the system malloc().
86: */
87: const int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
91: ptr = (void*)(((int*)ptr) - shift);
92: }
93: # endif
95: # if PetscDefined(HAVE_FREE_RETURN_INT)
96: int err = free(ptr);
98: # else
99: free(ptr);
100: # endif
101: #endif
102: return 0;
103: }
105: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
106: {
107: if (!mem) {
108: PetscFreeAlign(*result, line, func, file);
109: *result = NULL;
110: return 0;
111: }
112: #if PetscDefined(HAVE_MEMKIND)
113: *result = memkind_realloc(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,*result,mem);
114: #else
115: # if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_MEMALIGN))
116: {
117: /*
118: Previous int tells us how many ints the pointer has been shifted from
119: the original address provided by the system malloc().
120: */
121: int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
124: *result = (void*)(((int*)*result) - shift);
125: }
126: # endif
128: # if (PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || PetscDefined(HAVE_MEMALIGN)
129: *result = realloc(*result, mem);
130: # else
131: {
132: /*
133: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
134: */
135: int *ptr = (int*)realloc(*result,mem + 2*PETSC_MEMALIGN);
136: if (ptr) {
137: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
138: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
139: ptr[shift-1] = shift + SHIFT_CLASSID;
140: ptr += shift;
141: *result = (void*)ptr;
142: } else {
143: *result = NULL;
144: }
145: }
146: # endif
147: #endif
149: #if PetscDefined(HAVE_MEMALIGN)
150: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
151: * realloc and, if the alignment is wrong, malloc/copy/free. */
152: if (((size_t) (*result)) % PETSC_MEMALIGN) {
153: void *newResult;
154: # if PetscDefined(HAVE_MEMKIND)
155: {
156: int err;
157: err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
159: if (err == ENOMEM) PetscInfo(NULL,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
160: }
161: # else
162: newResult = memalign(PETSC_MEMALIGN,mem);
163: # endif
165: PetscMemcpy(newResult,*result,mem);
166: # if PetscDefined(HAVE_FREE_RETURN_INT)
167: {
168: int err = free(*result);
170: }
171: # else
172: # if defined(PETSC_HAVE_MEMKIND)
173: memkind_free(0,*result);
174: # else
175: free(*result);
176: # endif
177: # endif
178: *result = newResult;
179: }
180: #endif
181: return 0;
182: }
184: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
185: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign;
186: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
188: PETSC_INTERN PetscBool petscsetmallocvisited;
189: PetscBool petscsetmallocvisited = PETSC_FALSE;
191: /*@C
192: PetscMallocSet - Sets the routines used to do mallocs and frees.
193: This routine MUST be called before PetscInitialize() and may be
194: called only once.
196: Not Collective
198: Input Parameters:
199: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
200: . ifree - the routine that provides the free
201: - iralloc - the routine that provides the realloc
203: Level: developer
205: @*/
206: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),
207: PetscErrorCode (*ifree)(void*,int,const char[],const char[]),
208: PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
209: {
211: PetscTrMalloc = imalloc;
212: PetscTrFree = ifree;
213: PetscTrRealloc = iralloc;
214: petscsetmallocvisited = PETSC_TRUE;
215: return 0;
216: }
218: /*@C
219: PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.
221: Not Collective
223: Level: developer
225: Notes:
226: In general one should never run a PETSc program with different malloc() and
227: free() settings for different parts; this is because one NEVER wants to
228: free() an address that was malloced by a different memory management system
230: Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information
232: @*/
233: PetscErrorCode PetscMallocClear(void)
234: {
235: PetscTrMalloc = PetscMallocAlign;
236: PetscTrFree = PetscFreeAlign;
237: PetscTrRealloc = PetscReallocAlign;
238: petscsetmallocvisited = PETSC_FALSE;
239: return 0;
240: }
242: PetscErrorCode PetscMemoryTrace(const char label[])
243: {
244: PetscLogDouble mem,mal;
245: static PetscLogDouble oldmem = 0,oldmal = 0;
247: PetscMemoryGetCurrentUsage(&mem);
248: PetscMallocGetCurrentUsage(&mal);
250: PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
251: oldmem = mem;
252: oldmal = mal;
253: return 0;
254: }
256: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
257: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
258: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign;
260: /*@C
261: PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
262: If memkind is available, change the memkind type. Otherwise, switch the
263: current malloc and free routines to the PetscMallocAlign and
264: PetscFreeAlign (PETSc default).
266: Not Collective
268: Level: developer
270: Notes:
271: This provides a way to do the allocation on DRAM temporarily. One
272: can switch back to the previous choice by calling PetscMallocReset().
274: .seealso: PetscMallocReset()
275: @*/
276: PetscErrorCode PetscMallocSetDRAM(void)
277: {
278: if (PetscTrMalloc == PetscMallocAlign) {
279: #if defined(PETSC_HAVE_MEMKIND)
280: previousmktype = currentmktype;
281: currentmktype = PETSC_MK_DEFAULT;
282: #endif
283: } else {
284: /* Save the previous choice */
285: PetscTrMallocOld = PetscTrMalloc;
286: PetscTrReallocOld = PetscTrRealloc;
287: PetscTrFreeOld = PetscTrFree;
288: PetscTrMalloc = PetscMallocAlign;
289: PetscTrFree = PetscFreeAlign;
290: PetscTrRealloc = PetscReallocAlign;
291: }
292: return 0;
293: }
295: /*@C
296: PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM
298: Not Collective
300: Level: developer
302: .seealso: PetscMallocSetDRAM()
303: @*/
304: PetscErrorCode PetscMallocResetDRAM(void)
305: {
306: if (PetscTrMalloc == PetscMallocAlign) {
307: #if defined(PETSC_HAVE_MEMKIND)
308: currentmktype = previousmktype;
309: #endif
310: } else {
311: /* Reset to the previous choice */
312: PetscTrMalloc = PetscTrMallocOld;
313: PetscTrRealloc = PetscTrReallocOld;
314: PetscTrFree = PetscTrFreeOld;
315: }
316: return 0;
317: }
319: static PetscBool petscmalloccoalesce =
320: #if defined(PETSC_USE_MALLOC_COALESCED)
321: PETSC_TRUE;
322: #else
323: PETSC_FALSE;
324: #endif
326: /*@C
327: PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects
329: Not Collective
331: Input Parameters:
332: . coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.
334: Options Database Keys:
335: . -malloc_coalesce - turn coalesced malloc on or off
337: Note:
338: PETSc uses coalesced malloc by default for optimized builds and not for debugging builds. This default can be changed via the command-line option -malloc_coalesce or by calling this function.
339: This function can only be called immediately after PetscInitialize()
341: Level: developer
343: .seealso: PetscMallocA()
344: @*/
345: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
346: {
347: petscmalloccoalesce = coalesce;
348: return 0;
349: }
351: /*@C
352: PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc
354: Not Collective
356: Input Parameters:
357: + n - number of objects to allocate (at least 1)
358: . clear - use calloc() to allocate space initialized to zero
359: . lineno - line number to attribute allocation (typically __LINE__)
360: . function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
361: . filename - file name to attribute allocation (typically __FILE__)
362: - bytes0 - first of n object sizes
364: Output Parameters:
365: . ptr0 - first of n pointers to allocate
367: Notes:
368: This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.
370: Level: developer
372: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
373: @*/
374: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
375: {
376: va_list Argp;
377: size_t bytes[8],sumbytes;
378: void **ptr[8];
379: int i;
382: bytes[0] = bytes0;
383: ptr[0] = (void**)ptr0;
384: sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
385: va_start(Argp,ptr0);
386: for (i=1; i<n; i++) {
387: bytes[i] = va_arg(Argp,size_t);
388: ptr[i] = va_arg(Argp,void**);
389: sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
390: }
391: va_end(Argp);
392: if (petscmalloccoalesce) {
393: char *p;
394: (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
395: if (p == NULL) {
396: for (i=0; i<n; i++) {
397: *ptr[i] = NULL;
398: }
399: } else {
400: for (i=0; i<n; i++) {
401: *ptr[i] = bytes[i] ? p : NULL;
402: p = (char*)PetscAddrAlign(p + bytes[i]);
403: }
404: }
405: } else {
406: for (i=0; i<n; i++) {
407: (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
408: }
409: }
410: return 0;
411: }
413: /*@C
414: PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc
416: Not Collective
418: Input Parameters:
419: + n - number of objects to free (at least 1)
420: . lineno - line number to attribute deallocation (typically __LINE__)
421: . function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
422: . filename - file name to attribute deallocation (typically __FILE__)
423: - ptr0 ... - first of n pointers to free
425: Note:
426: This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.
428: The pointers are zeroed to prevent users from accidentally reusing space that has been freed.
430: Level: developer
432: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
433: @*/
434: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
435: {
436: va_list Argp;
437: void **ptr[8];
438: int i;
441: ptr[0] = (void**)ptr0;
442: va_start(Argp,ptr0);
443: for (i=1; i<n; i++) {
444: ptr[i] = va_arg(Argp,void**);
445: }
446: va_end(Argp);
447: if (petscmalloccoalesce) {
448: for (i=0; i<n; i++) { /* Find first nonempty allocation */
449: if (*ptr[i]) break;
450: }
451: while (--n > i) {
452: *ptr[n] = NULL;
453: }
454: (*PetscTrFree)(*ptr[n],lineno,function,filename);
455: *ptr[n] = NULL;
456: } else {
457: while (--n >= 0) {
458: (*PetscTrFree)(*ptr[n],lineno,function,filename);
459: *ptr[n] = NULL;
460: }
461: }
462: return 0;
463: }