Actual source code: mal.c
petsc-3.9.4 2018-09-11
2: /*
3: Code that allows a user to dictate what malloc() PETSc uses.
4: */
5: #include <petscsys.h>
6: #include <stdarg.h>
7: #if defined(PETSC_HAVE_MALLOC_H)
8: #include <malloc.h>
9: #endif
10: #if defined(PETSC_HAVE_MEMKIND)
11: #include <errno.h>
12: #include <memkind.h>
13: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
14: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
15: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
16: #endif
17: /*
18: We want to make sure that all mallocs of double or complex numbers are complex aligned.
19: 1) on systems with memalign() we call that routine to get an aligned memory location
20: 2) on systems without memalign() we
21: - allocate one sizeof(PetscScalar) extra space
22: - we shift the pointer up slightly if needed to get PetscScalar aligned
23: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
24: */
25: #define SHIFT_CLASSID 456123
27: PetscErrorCode PetscMallocAlign(size_t mem,int line,const char func[],const char file[],void **result)
28: {
29: if (!mem) { *result = NULL; return 0; }
30: #if defined(PETSC_HAVE_MEMKIND)
31: {
32: int ierr;
33: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
34: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
35: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
36: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
37: }
38: #else
39: # if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
40: *result = malloc(mem);
41: # elif defined(PETSC_HAVE_MEMALIGN)
42: *result = memalign(PETSC_MEMALIGN,mem);
43: # else
44: {
45: /*
46: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
47: */
48: int *ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
49: if (ptr) {
50: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
51: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
52: ptr[shift-1] = shift + SHIFT_CLASSID;
53: ptr += shift;
54: *result = (void*)ptr;
55: } else {
56: *result = NULL;
57: }
58: }
59: # endif
60: #endif
61: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
62: return 0;
63: }
65: PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
66: {
67: if (!ptr) return 0;
68: #if defined(PETSC_HAVE_MEMKIND)
69: memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
70: #else
71: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
72: {
73: /*
74: Previous int tells us how many ints the pointer has been shifted from
75: the original address provided by the system malloc().
76: */
77: int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
78: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
79: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
80: ptr = (void*)(((int*)ptr) - shift);
81: }
82: # endif
84: # if defined(PETSC_HAVE_FREE_RETURN_INT)
85: int err = free(ptr);
86: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
87: # else
88: free(ptr);
89: # endif
90: #endif
91: return 0;
92: }
94: PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
95: {
98: if (!mem) {
99: PetscFreeAlign(*result, line, func, file);
100: if (ierr) return ierr;
101: *result = NULL;
102: return 0;
103: }
104: #if defined(PETSC_HAVE_MEMKIND)
105: if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
106: else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
107: #else
108: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
109: {
110: /*
111: Previous int tells us how many ints the pointer has been shifted from
112: the original address provided by the system malloc().
113: */
114: int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
115: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
116: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
117: *result = (void*)(((int*)*result) - shift);
118: }
119: # endif
121: # if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
122: *result = realloc(*result, mem);
123: # else
124: {
125: /*
126: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
127: */
128: int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
129: if (ptr) {
130: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
131: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
132: ptr[shift-1] = shift + SHIFT_CLASSID;
133: ptr += shift;
134: *result = (void*)ptr;
135: } else {
136: *result = NULL;
137: }
138: }
139: # endif
140: #endif
141: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
142: #if defined(PETSC_HAVE_MEMALIGN)
143: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
144: * realloc and, if the alignment is wrong, malloc/copy/free. */
145: if (((size_t) (*result)) % PETSC_MEMALIGN) {
146: void *newResult;
147: # if defined(PETSC_HAVE_MEMKIND)
148: {
149: int ierr;
150: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
151: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
152: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
153: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
154: }
155: # else
156: newResult = memalign(PETSC_MEMALIGN,mem);
157: # endif
158: if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
159: PetscMemcpy(newResult,*result,mem);
160: if (ierr) return ierr;
161: # if defined(PETSC_HAVE_FREE_RETURN_INT)
162: {
163: int err = free(*result);
164: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
165: }
166: # else
167: # if defined(PETSC_HAVE_MEMKIND)
168: memkind_free(0,*result);
169: # else
170: free(*result);
171: # endif
172: # endif
173: *result = newResult;
174: }
175: #endif
176: return 0;
177: }
179: PetscErrorCode (*PetscTrMalloc)(size_t,int,const char[],const char[],void**) = PetscMallocAlign;
180: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign;
181: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
183: PetscBool petscsetmallocvisited = PETSC_FALSE;
185: /*@C
186: PetscMallocSet - Sets the routines used to do mallocs and frees.
187: This routine MUST be called before PetscInitialize() and may be
188: called only once.
190: Not Collective
192: Input Parameters:
193: + malloc - the malloc routine
194: - free - the free routine
196: Level: developer
198: Concepts: malloc
199: Concepts: memory^allocation
201: @*/
202: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,int,const char[],const char[],void**),
203: PetscErrorCode (*ifree)(void*,int,const char[],const char[]))
204: {
206: if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
207: PetscTrMalloc = imalloc;
208: PetscTrFree = ifree;
209: petscsetmallocvisited = PETSC_TRUE;
210: return(0);
211: }
213: /*@C
214: PetscMallocClear - Resets the routines used to do mallocs and frees to the
215: defaults.
217: Not Collective
219: Level: developer
221: Notes:
222: In general one should never run a PETSc program with different malloc() and
223: free() settings for different parts; this is because one NEVER wants to
224: free() an address that was malloced by a different memory management system
226: @*/
227: PetscErrorCode PetscMallocClear(void)
228: {
230: PetscTrMalloc = PetscMallocAlign;
231: PetscTrFree = PetscFreeAlign;
232: petscsetmallocvisited = PETSC_FALSE;
233: return(0);
234: }
236: PetscErrorCode PetscMemoryTrace(const char label[])
237: {
238: PetscErrorCode ierr;
239: PetscLogDouble mem,mal;
240: static PetscLogDouble oldmem = 0,oldmal = 0;
243: PetscMemoryGetCurrentUsage(&mem);
244: PetscMallocGetCurrentUsage(&mal);
246: PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
247: oldmem = mem;
248: oldmal = mal;
249: return(0);
250: }
252: static PetscErrorCode (*PetscTrMallocOld)(size_t,int,const char[],const char[],void**) = PetscMallocAlign;
253: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign;
255: /*@C
256: PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
257: If memkind is available, change the memkind type. Otherwise, switch the
258: current malloc and free routines to the PetscMallocAlign and
259: PetscFreeAlign (PETSc default).
261: Not Collective
263: Level: developer
265: Notes:
266: This provides a way to do the allocation on DRAM temporarily. One
267: can switch back to the previous choice by calling PetscMallocReset().
268:
269: .seealso: PetscMallocReset()
270: @*/
271: PetscErrorCode PetscMallocSetDRAM(void)
272: {
274: if (PetscTrMalloc == PetscMallocAlign) {
275: #if defined(PETSC_HAVE_MEMKIND)
276: previousmktype = currentmktype;
277: currentmktype = PETSC_MK_DEFAULT;
278: #endif
279: } else {
280: /* Save the previous choice */
281: PetscTrMallocOld = PetscTrMalloc;
282: PetscTrFreeOld = PetscTrFree;
283: PetscTrMalloc = PetscMallocAlign;
284: PetscTrFree = PetscFreeAlign;
285: }
286: return(0);
287: }
289: /*@C
290: PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM
292: Not Collective
294: Level: developer
296: .seealso: PetscMallocSetDRAM()
297: @*/
298: PetscErrorCode PetscMallocResetDRAM(void)
299: {
301: if (PetscTrMalloc == PetscMallocAlign) {
302: #if defined(PETSC_HAVE_MEMKIND)
303: currentmktype = previousmktype;
304: #endif
305: } else {
306: /* Reset to the previous choice */
307: PetscTrMalloc = PetscTrMallocOld;
308: PetscTrFree = PetscTrFreeOld;
309: }
310: return(0);
311: }
313: static PetscBool petscmalloccoalesce =
314: #if defined(PETSC_USE_MALLOC_COALESCED)
315: PETSC_TRUE;
316: #else
317: PETSC_FALSE;
318: #endif
320: /*@C
321: PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects
323: Not Collective
325: Input Parameters:
326: . coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.
328: Options Database Keys:
329: . -malloc_coalesce - turn coalesced malloc on or off
331: Note:
332: PETSc uses coalesced malloc by default for optimized builds and not for debugging builds. This default can be changed via the command-line option -malloc_coalesce or by calling this function.
334: Level: developer
336: .seealso: PetscMallocA()
337: @*/
338: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
339: {
341: petscmalloccoalesce = coalesce;
342: return(0);
343: }
345: /*@C
346: PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc
348: Not Collective
350: Input Parameters:
351: + n - number of objects to allocate (at least 1)
352: . clear - use calloc() to allocate space initialized to zero
353: . lineno - line number to attribute allocation (typically __LINE__)
354: . function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
355: . filename - file name to attribute allocation (typically __FILE__)
356: - bytes0 - first of n object sizes
358: Output Parameters:
359: . ptr0 - first of n pointers to allocate
361: Notes:
362: This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.
364: Level: developer
366: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
367: @*/
368: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
369: {
371: va_list Argp;
372: size_t bytes[8],sumbytes;
373: void **ptr[8];
374: int i;
377: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
378: bytes[0] = bytes0;
379: ptr[0] = (void**)ptr0;
380: sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
381: va_start(Argp,ptr0);
382: for (i=1; i<n; i++) {
383: bytes[i] = va_arg(Argp,size_t);
384: ptr[i] = va_arg(Argp,void**);
385: sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
386: }
387: va_end(Argp);
388: if (petscmalloccoalesce) {
389: char *p;
390: (*PetscTrMalloc)(sumbytes,lineno,function,filename,(void**)&p);
391: for (i=0; i<n; i++) {
392: *ptr[i] = bytes[i] ? p : NULL;
393: p = (char*)PetscAddrAlign(p + bytes[i]);
394: }
395: } else {
396: for (i=0; i<n; i++) {
397: (*PetscTrMalloc)(bytes[i],lineno,function,filename,(void**)ptr[i]);
398: }
399: }
400: if (clear) {
401: for (i=0; i<n; i++) {
402: PetscMemzero(*ptr[i],bytes[i]);
403: }
404: }
405: return(0);
406: }
408: /*@C
409: PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc
411: Not Collective
413: Input Parameters:
414: + n - number of objects to free (at least 1)
415: . lineno - line number to attribute deallocation (typically __LINE__)
416: . function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
417: . filename - file name to attribute deallocation (typically __FILE__)
418: - ptr0 ... - first of n pointers to free
420: Note:
421: This function is not normally called directly by users, but rather via the macros PetscFree1(), PetscFree2(), etc.
423: The pointers are zeroed to prevent users from accidently reusing space that has been freed.
425: Level: developer
427: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
428: @*/
429: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
430: {
432: va_list Argp;
433: void **ptr[8];
434: int i;
437: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
438: ptr[0] = (void**)ptr0;
439: va_start(Argp,ptr0);
440: for (i=1; i<n; i++) {
441: ptr[i] = va_arg(Argp,void**);
442: }
443: va_end(Argp);
444: if (petscmalloccoalesce) {
445: for (i=0; i<n; i++) { /* Find first nonempty allocation */
446: if (*ptr[i]) break;
447: }
448: while (--n > i) {
449: *ptr[n] = NULL;
450: }
451: (*PetscTrFree)(*ptr[n],lineno,function,filename);
452: *ptr[n] = NULL;
453: } else {
454: while (--n >= 0) {
455: (*PetscTrFree)(*ptr[n],lineno,function,filename);
456: *ptr[n] = NULL;
457: }
458: }
459: return(0);
460: }