1: /*
2: Code that allows a user to dictate what malloc() PETSc uses.
3: */
4: #include <petscsys.h> 5: #include <stdarg.h>
6: #if defined(PETSC_HAVE_MALLOC_H)
7: #include <malloc.h>
8: #endif
9: #if defined(PETSC_HAVE_MEMKIND)
10: #include <errno.h>
11: #include <memkind.h>
12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
15: #endif
16: /*
17: We want to make sure that all mallocs of double or complex numbers are complex aligned.
18: 1) on systems with memalign() we call that routine to get an aligned memory location
19: 2) on systems without memalign() we
20: - allocate one sizeof(PetscScalar) extra space
21: - we shift the pointer up slightly if needed to get PetscScalar aligned
22: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
23: */
24: #define SHIFT_CLASSID 456123 26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result) 27: {
30: if (!mem) {*result = NULL; return 0;}
31: #if defined(PETSC_HAVE_MEMKIND)
32: {
33: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
34: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
35: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
36: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
37: if (clear) {PetscMemzero(*result,mem);}
38: }
39: #else
40: # if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
41: if (clear) {
42: *result = calloc(1+mem/sizeof(int),sizeof(int));
43: } else {
44: *result = malloc(mem);
45: }
46: if (PetscLogMemory) {PetscMemzero(*result,mem);}
48: # elif defined(PETSC_HAVE_MEMALIGN)
49: *result = memalign(PETSC_MEMALIGN,mem);
50: if (clear || PetscLogMemory) {
51: PetscMemzero(*result,mem);
52: }
53: # else
54: {
55: int *ptr;
56: /*
57: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
58: */
59: if (clear) {
60: ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
61: } else {
62: ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
63: }
64: if (ptr) {
65: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
66: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
67: ptr[shift-1] = shift + SHIFT_CLASSID;
68: ptr += shift;
69: *result = (void*)ptr;
70: if (PetscLogMemory) {PetscMemzero(*result,mem);}
71: } else {
72: *result = NULL;
73: }
74: }
75: # endif
76: #endif
78: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
79: return 0;
80: }
82: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[]) 83: {
84: if (!ptr) return 0;
85: #if defined(PETSC_HAVE_MEMKIND)
86: memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
87: #else
88: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
89: {
90: /*
91: Previous int tells us how many ints the pointer has been shifted from
92: the original address provided by the system malloc().
93: */
94: int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
95: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
96: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
97: ptr = (void*)(((int*)ptr) - shift);
98: }
99: # endif
101: # if defined(PETSC_HAVE_FREE_RETURN_INT)
102: int err = free(ptr);
103: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
104: # else
105: free(ptr);
106: # endif
107: #endif
108: return 0;
109: }
111: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)112: {
115: if (!mem) {
116: PetscFreeAlign(*result, line, func, file);
117: if (ierr) return ierr;
118: *result = NULL;
119: return 0;
120: }
121: #if defined(PETSC_HAVE_MEMKIND)
122: if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
123: else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
124: #else
125: # if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
126: {
127: /*
128: Previous int tells us how many ints the pointer has been shifted from
129: the original address provided by the system malloc().
130: */
131: int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
132: if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
133: if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
134: *result = (void*)(((int*)*result) - shift);
135: }
136: # endif
138: # if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
139: *result = realloc(*result, mem);
140: # else
141: {
142: /*
143: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
144: */
145: int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
146: if (ptr) {
147: int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
148: shift = (2*PETSC_MEMALIGN - shift)/sizeof(int);
149: ptr[shift-1] = shift + SHIFT_CLASSID;
150: ptr += shift;
151: *result = (void*)ptr;
152: } else {
153: *result = NULL;
154: }
155: }
156: # endif
157: #endif
158: if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
159: #if defined(PETSC_HAVE_MEMALIGN)
160: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
161: * realloc and, if the alignment is wrong, malloc/copy/free. */
162: if (((size_t) (*result)) % PETSC_MEMALIGN) {
163: void *newResult;
164: # if defined(PETSC_HAVE_MEMKIND)
165: {
166: int ierr;
167: if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
168: else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
169: if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
170: if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
171: }
172: # else
173: newResult = memalign(PETSC_MEMALIGN,mem);
174: # endif
175: if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
176: PetscMemcpy(newResult,*result,mem);
177: if (ierr) return ierr;
178: # if defined(PETSC_HAVE_FREE_RETURN_INT)
179: {
180: int err = free(*result);
181: if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
182: }
183: # else
184: # if defined(PETSC_HAVE_MEMKIND)
185: memkind_free(0,*result);
186: # else
187: free(*result);
188: # endif
189: # endif
190: *result = newResult;
191: }
192: #endif
193: return 0;
194: }
196: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
197: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign;
198: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
200: PETSC_INTERN PetscBool petscsetmallocvisited;
201: PetscBool petscsetmallocvisited = PETSC_FALSE;
203: /*@C
204: PetscMallocSet - Sets the routines used to do mallocs and frees.
205: This routine MUST be called before PetscInitialize() and may be
206: called only once.
208: Not Collective
210: Input Parameters:
211: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
212: . ifree - the routine that provides the free
213: - iralloc - the routine that provides the realloc
215: Level: developer
217: @*/
218: PetscErrorCodePetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),219: PetscErrorCode (*ifree)(void*,int,const char[],const char[]),220: PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))221: {
223: if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
224: PetscTrMalloc = imalloc;
225: PetscTrFree = ifree;
226: PetscTrRealloc = iralloc;
227: petscsetmallocvisited = PETSC_TRUE;
228: return(0);
229: }
231: /*@C
232: PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.
234: Not Collective
236: Level: developer
238: Notes:
239: In general one should never run a PETSc program with different malloc() and
240: free() settings for different parts; this is because one NEVER wants to
241: free() an address that was malloced by a different memory management system
243: Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information
245: @*/
246: PetscErrorCodePetscMallocClear(void)247: {
249: PetscTrMalloc = PetscMallocAlign;
250: PetscTrFree = PetscFreeAlign;
251: PetscTrRealloc = PetscReallocAlign;
252: petscsetmallocvisited = PETSC_FALSE;
253: return(0);
254: }
256: PetscErrorCode PetscMemoryTrace(const char label[])257: {
258: PetscErrorCode ierr;
259: PetscLogDouble mem,mal;
260: static PetscLogDouble oldmem = 0,oldmal = 0;
263: PetscMemoryGetCurrentUsage(&mem);
264: PetscMallocGetCurrentUsage(&mal);
266: PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
267: oldmem = mem;
268: oldmal = mal;
269: return(0);
270: }
272: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
273: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**) = PetscReallocAlign;
274: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign;
276: /*@C
277: PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
278: If memkind is available, change the memkind type. Otherwise, switch the
279: current malloc and free routines to the PetscMallocAlign and
280: PetscFreeAlign (PETSc default).
282: Not Collective
284: Level: developer
286: Notes:
287: This provides a way to do the allocation on DRAM temporarily. One
288: can switch back to the previous choice by calling PetscMallocReset().
290: .seealso: PetscMallocReset()
291: @*/
292: PetscErrorCodePetscMallocSetDRAM(void)293: {
295: if (PetscTrMalloc == PetscMallocAlign) {
296: #if defined(PETSC_HAVE_MEMKIND)
297: previousmktype = currentmktype;
298: currentmktype = PETSC_MK_DEFAULT;
299: #endif
300: } else {
301: /* Save the previous choice */
302: PetscTrMallocOld = PetscTrMalloc;
303: PetscTrReallocOld = PetscTrRealloc;
304: PetscTrFreeOld = PetscTrFree;
305: PetscTrMalloc = PetscMallocAlign;
306: PetscTrFree = PetscFreeAlign;
307: PetscTrRealloc = PetscReallocAlign;
308: }
309: return(0);
310: }
312: /*@C
313: PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM315: Not Collective
317: Level: developer
319: .seealso: PetscMallocSetDRAM()
320: @*/
321: PetscErrorCodePetscMallocResetDRAM(void)322: {
324: if (PetscTrMalloc == PetscMallocAlign) {
325: #if defined(PETSC_HAVE_MEMKIND)
326: currentmktype = previousmktype;
327: #endif
328: } else {
329: /* Reset to the previous choice */
330: PetscTrMalloc = PetscTrMallocOld;
331: PetscTrRealloc = PetscTrReallocOld;
332: PetscTrFree = PetscTrFreeOld;
333: }
334: return(0);
335: }
337: static PetscBool petscmalloccoalesce =
338: #if defined(PETSC_USE_MALLOC_COALESCED)
339: PETSC_TRUE;
340: #else
341: PETSC_FALSE;
342: #endif
344: /*@C
345: PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects
347: Not Collective
349: Input Parameters:
350: . coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.
352: Options Database Keys:
353: . -malloc_coalesce - turn coalesced malloc on or off
355: Note:
356: PETSc uses coalesced malloc by default for optimized builds and not for debugging builds. This default can be changed via the command-line option -malloc_coalesce or by calling this function.
357: This function can only be called immediately after PetscInitialize()
359: Level: developer
361: .seealso: PetscMallocA()
362: @*/
363: PetscErrorCodePetscMallocSetCoalesce(PetscBool coalesce)364: {
366: petscmalloccoalesce = coalesce;
367: return(0);
368: }
370: /*@C
371: PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc
373: Not Collective
375: Input Parameters:
376: + n - number of objects to allocate (at least 1)
377: . clear - use calloc() to allocate space initialized to zero
378: . lineno - line number to attribute allocation (typically __LINE__)
379: . function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
380: . filename - file name to attribute allocation (typically __FILE__)
381: - bytes0 - first of n object sizes
383: Output Parameters:
384: . ptr0 - first of n pointers to allocate
386: Notes:
387: This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.
389: Level: developer
391: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
392: @*/
393: PetscErrorCodePetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)394: {
396: va_list Argp;
397: size_t bytes[8],sumbytes;
398: void **ptr[8];
399: int i;
402: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
403: bytes[0] = bytes0;
404: ptr[0] = (void**)ptr0;
405: sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
406: va_start(Argp,ptr0);
407: for (i=1; i<n; i++) {
408: bytes[i] = va_arg(Argp,size_t);
409: ptr[i] = va_arg(Argp,void**);
410: sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
411: }
412: va_end(Argp);
413: if (petscmalloccoalesce) {
414: char *p;
415: (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
416: for (i=0; i<n; i++) {
417: *ptr[i] = bytes[i] ? p : NULL;
418: p = (char*)PetscAddrAlign(p + bytes[i]);
419: }
420: } else {
421: for (i=0; i<n; i++) {
422: (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
423: }
424: }
425: return(0);
426: }
428: /*@C
429: PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc
431: Not Collective
433: Input Parameters:
434: + n - number of objects to free (at least 1)
435: . lineno - line number to attribute deallocation (typically __LINE__)
436: . function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
437: . filename - file name to attribute deallocation (typically __FILE__)
438: - ptr0 ... - first of n pointers to free
440: Note:
441: This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.
443: The pointers are zeroed to prevent users from accidently reusing space that has been freed.
445: Level: developer
447: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
448: @*/
449: PetscErrorCodePetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)450: {
452: va_list Argp;
453: void **ptr[8];
454: int i;
457: if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
458: ptr[0] = (void**)ptr0;
459: va_start(Argp,ptr0);
460: for (i=1; i<n; i++) {
461: ptr[i] = va_arg(Argp,void**);
462: }
463: va_end(Argp);
464: if (petscmalloccoalesce) {
465: for (i=0; i<n; i++) { /* Find first nonempty allocation */
466: if (*ptr[i]) break;
467: }
468: while (--n > i) {
469: *ptr[n] = NULL;
470: }
471: (*PetscTrFree)(*ptr[n],lineno,function,filename);
472: *ptr[n] = NULL;
473: } else {
474: while (--n >= 0) {
475: (*PetscTrFree)(*ptr[n],lineno,function,filename);
476: *ptr[n] = NULL;
477: }
478: }
479: return(0);
480: }