Actual source code: mal.c
petsc-3.11.4 2019-09-28
1: /*
2: Code that allows a user to dictate what malloc() PETSc uses.
3: */
4: #include <petscsys.h>
5: #include <stdarg.h>
6: #if defined(PETSC_HAVE_MALLOC_H)
7: #include <malloc.h>
8: #endif
9: #if defined(PETSC_HAVE_MEMKIND)
10: #include <errno.h>
11: #include <memkind.h>
12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
15: #endif
16: /*
17: We want to make sure that all mallocs of double or complex numbers are complex aligned.
18: 1) on systems with memalign() we call that routine to get an aligned memory location
19: 2) on systems without memalign() we
20: - allocate one sizeof(PetscScalar) extra space
21: - we shift the pointer up slightly if needed to get PetscScalar aligned
22: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
23: */
24: #define SHIFT_CLASSID 456123
26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,int line,const char func[],const char file[],void **result)
27: {
28: if (!mem) { *result = NULL; return 0; }
29: #if defined(PETSC_HAVE_MEMKIND)
30: {
31: int ierr;
32: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
33: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
34: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
35: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
36: }
37: #else
38: # if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
39: *result = malloc(mem);
40: # elif defined(PETSC_HAVE_MEMALIGN)
41: *result = memalign(PETSC_MEMALIGN,mem);
42: # else
43: {
44: /*
45: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
46: */
47: int *ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
48: if (ptr) {
49: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
50: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
51: ptr[shift-1] = shift + SHIFT_CLASSID;
52: ptr += shift;
53: *result = (void*)ptr;
54: } else {
55: *result = NULL;
56: }
57: }
58: # endif
59: #endif
60: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
61: return 0;
62: }
64: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
65: {
66: if (!ptr) return 0;
67: #if defined(PETSC_HAVE_MEMKIND)
68: memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
69: #else
70: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
71: {
72: /*
73: Previous int tells us how many ints the pointer has been shifted from
74: the original address provided by the system malloc().
75: */
76: int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
77: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
78: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
79: ptr = (void*)(((int*)ptr) - shift);
80: }
81: # endif
83: # if defined(PETSC_HAVE_FREE_RETURN_INT)
84: int err = free(ptr);
85: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
86: # else
87: free(ptr);
88: # endif
89: #endif
90: return 0;
91: }
93: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
94: {
97: if (!mem) {
98: PetscFreeAlign(*result, line, func, file);
99: if (ierr) return ierr;
100: *result = NULL;
101: return 0;
102: }
103: #if defined(PETSC_HAVE_MEMKIND)
104: if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
105: else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
106: #else
107: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
108: {
109: /*
110: Previous int tells us how many ints the pointer has been shifted from
111: the original address provided by the system malloc().
112: */
113: int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
114: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
115: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
116: *result = (void*)(((int*)*result) - shift);
117: }
118: # endif
120: # if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
121: *result = realloc(*result, mem);
122: # else
123: {
124: /*
125: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
126: */
127: int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
128: if (ptr) {
129: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
130: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
131: ptr[shift-1] = shift + SHIFT_CLASSID;
132: ptr += shift;
133: *result = (void*)ptr;
134: } else {
135: *result = NULL;
136: }
137: }
138: # endif
139: #endif
140: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
141: #if defined(PETSC_HAVE_MEMALIGN)
142: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
143: * realloc and, if the alignment is wrong, malloc/copy/free. */
144: if (((size_t) (*result)) % PETSC_MEMALIGN) {
145: void *newResult;
146: # if defined(PETSC_HAVE_MEMKIND)
147: {
148: int ierr;
149: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
150: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
151: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
152: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
153: }
154: # else
155: newResult = memalign(PETSC_MEMALIGN,mem);
156: # endif
157: if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
158: PetscMemcpy(newResult,*result,mem);
159: if (ierr) return ierr;
160: # if defined(PETSC_HAVE_FREE_RETURN_INT)
161: {
162: int err = free(*result);
163: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
164: }
165: # else
166: # if defined(PETSC_HAVE_MEMKIND)
167: memkind_free(0,*result);
168: # else
169: free(*result);
170: # endif
171: # endif
172: *result = newResult;
173: }
174: #endif
175: return 0;
176: }
178: PetscErrorCode (*PetscTrMalloc)(size_t,int,const char[],const char[],void**) = PetscMallocAlign;
179: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign;
180: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
182: PETSC_INTERN PetscBool petscsetmallocvisited;
183: PetscBool petscsetmallocvisited = PETSC_FALSE;
185: /*@C
186: PetscMallocSet - Sets the routines used to do mallocs and frees.
187: This routine MUST be called before PetscInitialize() and may be
188: called only once.
190: Not Collective
192: Input Parameters:
193: + malloc - the malloc routine
194: - free - the free routine
196: Level: developer
198: Concepts: malloc
199: Concepts: memory^allocation
201: @*/
202: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,int,const char[],const char[],void**),
203: PetscErrorCode (*ifree)(void*,int,const char[],const char[]))
204: {
206: if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
207: PetscTrMalloc = imalloc;
208: PetscTrFree = ifree;
209: petscsetmallocvisited = PETSC_TRUE;
210: return(0);
211: }
213: /*@C
214: PetscMallocClear - Resets the routines used to do mallocs and frees to the
215: defaults.
217: Not Collective
219: Level: developer
221: Notes:
222: In general one should never run a PETSc program with different malloc() and
223: free() settings for different parts; this is because one NEVER wants to
224: free() an address that was malloced by a different memory management system
226: @*/
227: PetscErrorCode PetscMallocClear(void)
228: {
230: PetscTrMalloc = PetscMallocAlign;
231: PetscTrFree = PetscFreeAlign;
232: petscsetmallocvisited = PETSC_FALSE;
233: return(0);
234: }
236: PetscErrorCode PetscMemoryTrace(const char label[])
237: {
238: PetscErrorCode ierr;
239: PetscLogDouble mem,mal;
240: static PetscLogDouble oldmem = 0,oldmal = 0;
243: PetscMemoryGetCurrentUsage(&mem);
244: PetscMallocGetCurrentUsage(&mal);
246: PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
247: oldmem = mem;
248: oldmal = mal;
249: return(0);
250: }
252: static PetscErrorCode (*PetscTrMallocOld)(size_t,int,const char[],const char[],void**) = PetscMallocAlign;
253: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign;
255: /*@C
256: PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
257: If memkind is available, change the memkind type. Otherwise, switch the
258: current malloc and free routines to the PetscMallocAlign and
259: PetscFreeAlign (PETSc default).
261: Not Collective
263: Level: developer
265: Notes:
266: This provides a way to do the allocation on DRAM temporarily. One
267: can switch back to the previous choice by calling PetscMallocReset().
269: .seealso: PetscMallocReset()
270: @*/
271: PetscErrorCode PetscMallocSetDRAM(void)
272: {
274: if (PetscTrMalloc == PetscMallocAlign) {
275: #if defined(PETSC_HAVE_MEMKIND)
276: previousmktype = currentmktype;
277: currentmktype = PETSC_MK_DEFAULT;
278: #endif
279: } else {
280: /* Save the previous choice */
281: PetscTrMallocOld = PetscTrMalloc;
282: PetscTrFreeOld = PetscTrFree;
283: PetscTrMalloc = PetscMallocAlign;
284: PetscTrFree = PetscFreeAlign;
285: }
286: return(0);
287: }
289: /*@C
290: PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM
292: Not Collective
294: Level: developer
296: .seealso: PetscMallocSetDRAM()
297: @*/
298: PetscErrorCode PetscMallocResetDRAM(void)
299: {
301: if (PetscTrMalloc == PetscMallocAlign) {
302: #if defined(PETSC_HAVE_MEMKIND)
303: currentmktype = previousmktype;
304: #endif
305: } else {
306: /* Reset to the previous choice */
307: PetscTrMalloc = PetscTrMallocOld;
308: PetscTrFree = PetscTrFreeOld;
309: }
310: return(0);
311: }
313: static PetscBool petscmalloccoalesce =
314: #if defined(PETSC_USE_MALLOC_COALESCED)
315: PETSC_TRUE;
316: #else
317: PETSC_FALSE;
318: #endif
320: /*@C
321: PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects
323: Not Collective
325: Input Parameters:
326: . coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.
328: Options Database Keys:
329: . -malloc_coalesce - turn coalesced malloc on or off
331: Note:
332: PETSc uses coalesced malloc by default for optimized builds and not for debugging builds. This default can be changed via the command-line option -malloc_coalesce or by calling this function.
334: Level: developer
336: .seealso: PetscMallocA()
337: @*/
338: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
339: {
341: petscmalloccoalesce = coalesce;
342: return(0);
343: }
345: /*@C
346: PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc
348: Not Collective
350: Input Parameters:
351: + n - number of objects to allocate (at least 1)
352: . clear - use calloc() to allocate space initialized to zero
353: . lineno - line number to attribute allocation (typically __LINE__)
354: . function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
355: . filename - file name to attribute allocation (typically __FILE__)
356: - bytes0 - first of n object sizes
358: Output Parameters:
359: . ptr0 - first of n pointers to allocate
361: Notes:
362: This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.
364: Level: developer
366: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
367: @*/
368: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
369: {
371: va_list Argp;
372: size_t bytes[8],sumbytes;
373: void **ptr[8];
374: int i;
377: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
378: bytes[0] = bytes0;
379: ptr[0] = (void**)ptr0;
380: sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
381: va_start(Argp,ptr0);
382: for (i=1; i<n; i++) {
383: bytes[i] = va_arg(Argp,size_t);
384: ptr[i] = va_arg(Argp,void**);
385: sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
386: }
387: va_end(Argp);
388: if (petscmalloccoalesce) {
389: char *p;
390: (*PetscTrMalloc)(sumbytes,lineno,function,filename,(void**)&p);
391: for (i=0; i<n; i++) {
392: *ptr[i] = bytes[i] ? p : NULL;
393: p = (char*)PetscAddrAlign(p + bytes[i]);
394: }
395: } else {
396: for (i=0; i<n; i++) {
397: (*PetscTrMalloc)(bytes[i],lineno,function,filename,(void**)ptr[i]);
398: }
399: }
400: if (clear) {
401: for (i=0; i<n; i++) {
402: PetscMemzero(*ptr[i],bytes[i]);
403: }
404: }
405: return(0);
406: }
408: /*@C
409: PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc
411: Not Collective
413: Input Parameters:
414: + n - number of objects to free (at least 1)
415: . lineno - line number to attribute deallocation (typically __LINE__)
416: . function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
417: . filename - file name to attribute deallocation (typically __FILE__)
418: - ptr0 ... - first of n pointers to free
420: Note:
421: This function is not normally called directly by users, but rather via the macros PetscFree1(), PetscFree2(), etc.
423: The pointers are zeroed to prevent users from accidently reusing space that has been freed.
425: Level: developer
427: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
428: @*/
429: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
430: {
432: va_list Argp;
433: void **ptr[8];
434: int i;
437: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
438: ptr[0] = (void**)ptr0;
439: va_start(Argp,ptr0);
440: for (i=1; i<n; i++) {
441: ptr[i] = va_arg(Argp,void**);
442: }
443: va_end(Argp);
444: if (petscmalloccoalesce) {
445: for (i=0; i<n; i++) { /* Find first nonempty allocation */
446: if (*ptr[i]) break;
447: }
448: while (--n > i) {
449: *ptr[n] = NULL;
450: }
451: (*PetscTrFree)(*ptr[n],lineno,function,filename);
452: *ptr[n] = NULL;
453: } else {
454: while (--n >= 0) {
455: (*PetscTrFree)(*ptr[n],lineno,function,filename);
456: *ptr[n] = NULL;
457: }
458: }
459: return(0);
460: }