Actual source code: init.c
petsc-3.14.6 2021-03-30
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot be known
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h>
10: #include <petsc/private/petscimpl.h>
11: #include <petscvalgrind.h>
12: #include <petscviewer.h>
13: #if defined(PETSC_USE_LOG)
14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
15: #endif
17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
18: #include <sys/sysinfo.h>
19: #endif
20: #if defined(PETSC_HAVE_UNISTD_H)
21: #include <unistd.h>
22: #endif
24: #if defined(PETSC_HAVE_CUDA)
25: #include <cuda_runtime.h>
26: #include <petsccublas.h>
27: #endif
29: #if defined(PETSC_HAVE_HIP)
30: #include <hip/hip_runtime.h>
31: #endif
33: #if defined(PETSC_HAVE_DEVICE)
34: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
35: #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
36: #endif
37: #endif
39: #if defined(PETSC_HAVE_VIENNACL)
40: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
41: #endif
44: /* ------------------------Nasty global variables -------------------------------*/
45: /*
46: Indicates if PETSc started up MPI, or it was
47: already started before PETSc was initialized.
48: */
49: PetscBool PetscBeganMPI = PETSC_FALSE;
50: PetscBool PetscErrorHandlingInitialized = PETSC_FALSE;
51: PetscBool PetscInitializeCalled = PETSC_FALSE;
52: PetscBool PetscFinalizeCalled = PETSC_FALSE;
54: PetscMPIInt PetscGlobalRank = -1;
55: PetscMPIInt PetscGlobalSize = -1;
57: #if defined(PETSC_HAVE_KOKKOS)
58: PetscBool PetscBeganKokkos = PETSC_FALSE;
59: #endif
61: PetscBool use_gpu_aware_mpi = PETSC_TRUE;
62: PetscBool PetscCreatedGpuObjects = PETSC_FALSE;
64: #if defined(PETSC_HAVE_COMPLEX)
65: #if defined(PETSC_COMPLEX_INSTANTIATE)
66: template <> class std::complex<double>; /* instantiate complex template class */
67: #endif
68: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
69: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
70: MPI_Datatype MPIU_C_COMPLEX;
71: #endif
73: /*MC
74: PETSC_i - the imaginary number i
76: Synopsis:
77: #include <petscsys.h>
78: PetscComplex PETSC_i;
80: Level: beginner
82: Note:
83: Complex numbers are automatically available if PETSc located a working complex implementation
85: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
86: M*/
87: PetscComplex PETSC_i;
88: #endif
89: #if defined(PETSC_USE_REAL___FLOAT128)
90: MPI_Datatype MPIU___FLOAT128 = 0;
91: #if defined(PETSC_HAVE_COMPLEX)
92: MPI_Datatype MPIU___COMPLEX128 = 0;
93: #endif
94: #elif defined(PETSC_USE_REAL___FP16)
95: MPI_Datatype MPIU___FP16 = 0;
96: #endif
97: MPI_Datatype MPIU_2SCALAR = 0;
98: #if defined(PETSC_USE_64BIT_INDICES)
99: MPI_Datatype MPIU_2INT = 0;
100: #endif
101: MPI_Datatype MPIU_BOOL;
102: MPI_Datatype MPIU_ENUM;
103: MPI_Datatype MPIU_FORTRANADDR;
104: MPI_Datatype MPIU_SIZE_T;
106: /*
107: Function that is called to display all error messages
108: */
109: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
110: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
111: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
112: /*
113: This is needed to turn on/off GPU synchronization
114: */
115: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
117: /* ------------------------------------------------------------------------------*/
118: /*
119: Optional file where all PETSc output from various prints is saved
120: */
121: PETSC_INTERN FILE *petsc_history;
122: FILE *petsc_history = NULL;
124: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
125: {
127: PetscMPIInt rank,size;
128: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
129: char version[256];
132: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
133: if (!rank) {
134: char arch[10];
135: int err;
137: PetscGetArchType(arch,10);
138: PetscGetDate(date,64);
139: PetscGetVersion(version,256);
140: MPI_Comm_size(PETSC_COMM_WORLD,&size);
141: if (filename) {
142: PetscFixFilename(filename,fname);
143: } else {
144: PetscGetHomeDirectory(pfile,sizeof(pfile));
145: PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
146: PetscFixFilename(pfile,fname);
147: }
149: *fd = fopen(fname,"a");
150: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
152: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
153: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
154: PetscGetProgramName(pname,sizeof(pname));
155: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
156: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
158: err = fflush(*fd);
159: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160: }
161: return(0);
162: }
164: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
165: {
167: PetscMPIInt rank;
168: char date[64];
169: int err;
172: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
173: if (!rank) {
174: PetscGetDate(date,64);
175: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
176: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
177: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
178: err = fflush(*fd);
179: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
180: err = fclose(*fd);
181: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
182: }
183: return(0);
184: }
186: /* ------------------------------------------------------------------------------*/
188: /*
189: This is ugly and probably belongs somewhere else, but I want to
190: be able to put a true MPI abort error handler with command line args.
192: This is so MPI errors in the debugger will leave all the stack
193: frames. The default MP_Abort() cleans up and exits thus providing no useful information
194: in the debugger hence we call abort() instead of MPI_Abort().
195: */
197: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
198: {
200: (*PetscErrorPrintf)("MPI error %d\n",*flag);
201: abort();
202: }
204: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
205: {
209: (*PetscErrorPrintf)("MPI error %d\n",*flag);
210: PetscAttachDebugger();
211: if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
212: }
214: /*@C
215: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
216: wishes a clean exit somewhere deep in the program.
218: Collective on PETSC_COMM_WORLD
220: Options Database Keys are the same as for PetscFinalize()
222: Level: advanced
224: Note:
225: See PetscInitialize() for more general runtime options.
227: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
228: @*/
229: PetscErrorCode PetscEnd(void)
230: {
232: PetscFinalize();
233: exit(0);
234: return 0;
235: }
237: PetscBool PetscOptionsPublish = PETSC_FALSE;
238: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
239: PETSC_INTERN PetscBool petscsetmallocvisited;
240: static char emacsmachinename[256];
242: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
243: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL;
245: /*@C
246: PetscSetHelpVersionFunctions - Sets functions that print help and version information
247: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
248: This routine enables a "higher-level" package that uses PETSc to print its messages first.
250: Input Parameter:
251: + help - the help function (may be NULL)
252: - version - the version function (may be NULL)
254: Level: developer
256: @*/
257: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
258: {
260: PetscExternalHelpFunction = help;
261: PetscExternalVersionFunction = version;
262: return(0);
263: }
265: #if defined(PETSC_USE_LOG)
266: PETSC_INTERN PetscBool PetscObjectsLog;
267: #endif
269: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
270: {
271: if (PetscUnlikely(*err)) {
272: PETSC_UNUSED PetscMPIInt len;
273: char errstring[MPI_MAX_ERROR_STRING];
275: MPI_Error_string(*err,errstring,&len);
276: PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
277: }
278: return;
279: }
281: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
282: Use the following macros to define CUDA/HIP initialization related vars/routines.
283: */
284: #if defined(PETSC_HAVE_CUDA)
285: typedef cudaError_t cupmError_t;
286: typedef struct cudaDeviceProp cupmDeviceProp;
287: #define cupmGetDeviceCount(x) cudaGetDeviceCount(x)
288: #define cupmGetDevice(x) cudaGetDevice(x)
289: #define cupmSetDevice(x) cudaSetDevice(x)
290: #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x)
291: #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y)
292: #define cupmGetLastError() cudaGetLastError()
293: #define cupmDeviceMapHost cudaDeviceMapHost
294: #define cupmSuccess cudaSuccess
295: #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation
296: #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources
297: #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess
298: #define CHKERRCUPM(x) CHKERRCUDA(x)
299: #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle()
300: #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle()
301: #define PetscCUPMInitialize PetscCUDAInitialize
302: #define PetscCUPMInitialized PetscCUDAInitialized
303: #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck
304: #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView
305: #define PetscCUPMSynchronize PetscCUDASynchronize
306: #define PetscNotUseCUPM PetscNotUseCUDA
307: #define cupmOptionsStr "CUDA options"
308: #define cupmSetDeviceStr "-cuda_device"
309: #define cupmViewStr "-cuda_view"
310: #define cupmSynchronizeStr "-cuda_synchronize"
311: #define PetscCUPMInitializeStr "PetscCUDAInitialize"
312: #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA
313: #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck
314: #include "cupminit.inc"
315: #endif
317: #if defined(PETSC_HAVE_HIP)
318: typedef hipError_t cupmError_t;
319: typedef hipDeviceProp_t cupmDeviceProp;
320: #define cupmGetDeviceCount(x) hipGetDeviceCount(x)
321: #define cupmGetDevice(x) hipGetDevice(x)
322: #define cupmSetDevice(x) hipSetDevice(x)
323: #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x)
324: #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y)
325: #define cupmGetLastError() hipGetLastError()
326: #define cupmDeviceMapHost hipDeviceMapHost
327: #define cupmSuccess hipSuccess
328: #define cupmErrorMemoryAllocation hipErrorMemoryAllocation
329: #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources
330: #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess
331: #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
332: #define PetscCUPMBLASInitializeHandle() 0
333: #define PetscCUPMSOLVERDnInitializeHandle() 0
334: #define PetscCUPMInitialize PetscHIPInitialize
335: #define PetscCUPMInitialized PetscHIPInitialized
336: #define PetscCUPMInitializeCheck PetscHIPInitializeCheck
337: #define PetscCUPMInitializeAndView PetscHIPInitializeAndView
338: #define PetscCUPMSynchronize PetscHIPSynchronize
339: #define PetscNotUseCUPM PetscNotUseHIP
340: #define cupmOptionsStr "HIP options"
341: #define cupmSetDeviceStr "-hip_device"
342: #define cupmViewStr "-hip_view"
343: #define cupmSynchronizeStr "-hip_synchronize"
344: #define PetscCUPMInitializeStr "PetscHIPInitialize"
345: #define PetscOptionsCheckCUPM PetscOptionsCheckHIP
346: #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck
347: #include "cupminit.inc"
348: #endif
350: PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[])
351: {
352: char string[64];
353: MPI_Comm comm = PETSC_COMM_WORLD;
354: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
355: PetscErrorCode ierr;
356: PetscReal si;
357: PetscInt intensity;
358: int i;
359: PetscMPIInt rank;
360: char version[256];
361: #if defined(PETSC_USE_LOG)
362: char mname[PETSC_MAX_PATH_LEN];
363: PetscViewerFormat format;
364: PetscBool flg4 = PETSC_FALSE;
365: #endif
368: MPI_Comm_rank(comm,&rank);
370: #if !defined(PETSC_HAVE_THREADSAFETY)
371: if (!(PETSC_RUNNING_ON_VALGRIND)) {
372: /*
373: Setup the memory management; support for tracing malloc() usage
374: */
375: PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;
377: if (PetscDefined(USE_DEBUG)) {
378: mdebug = PETSC_TRUE;
379: initializenan = PETSC_TRUE;
380: PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
381: } else {
382: /* don't warn about unused option */
383: PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
384: flg1 = PETSC_FALSE;
385: }
386: PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
387: if (flg1 || flg2) {
388: mdebug = PETSC_TRUE;
389: eachcall = PETSC_TRUE;
390: initializenan = PETSC_TRUE;
391: } else if (flg3 && !flg2) {
392: mdebug = PETSC_FALSE;
393: eachcall = PETSC_FALSE;
394: initializenan = PETSC_FALSE;
395: }
397: PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);
398: if (flg2) {PetscMallocLogRequestedSizeSet(flg1);}
400: PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
401: if (mlog) {
402: mdebug = PETSC_TRUE;
403: }
404: /* the next line is deprecated */
405: PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
406: PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
407: PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
408: if (mdebug) {
409: PetscMallocSetDebug(eachcall,initializenan);
410: }
411: if (mlog) {
412: PetscReal logthreshold = 0;
413: PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
414: PetscMallocViewSet(logthreshold);
415: }
416: #if defined(PETSC_USE_LOG)
417: PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
418: #endif
419: }
421: PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
422: if (flg2) {PetscMallocSetCoalesce(flg1);}
423: flg1 = PETSC_FALSE;
424: PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
425: /* ignore this option if malloc is already set */
426: if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}
428: flg1 = PETSC_FALSE;
429: PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
430: if (!flg1) {
431: flg1 = PETSC_FALSE;
432: PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
433: }
434: if (flg1) {
435: PetscMemorySetGetMaximumUsage();
436: }
437: #endif
439: #if defined(PETSC_USE_LOG)
440: PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
441: #endif
443: /*
444: Set the display variable for graphics
445: */
446: PetscSetDisplay();
448: /*
449: Print main application help message
450: */
451: PetscOptionsHasHelp(NULL,&hasHelp);
452: if (help && hasHelp) {
453: PetscPrintf(comm,help);
454: PetscPrintf(comm,"----------------------------------------\n");
455: }
457: /*
458: Print the PETSc version information
459: */
460: PetscOptionsHasName(NULL,NULL,"-version",&flg1);
461: if (flg1 || hasHelp) {
462: /*
463: Print "higher-level" package version message
464: */
465: if (PetscExternalVersionFunction) {
466: (*PetscExternalVersionFunction)(comm);
467: }
469: PetscGetVersion(version,256);
470: (*PetscHelpPrintf)(comm,"%s\n",version);
471: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
472: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
473: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
474: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
475: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
476: (*PetscHelpPrintf)(comm,"----------------------------------------\n");
477: }
479: /*
480: Print "higher-level" package help message
481: */
482: if (hasHelp) {
483: PetscBool hasHelpIntro;
485: if (PetscExternalHelpFunction) {
486: (*PetscExternalHelpFunction)(comm);
487: }
488: PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
489: if (hasHelpIntro) {
490: PetscOptionsDestroyDefault();
491: PetscFreeMPIResources();
492: MPI_Finalize();
493: exit(0);
494: }
495: }
497: /*
498: Setup the error handling
499: */
500: flg1 = PETSC_FALSE;
501: PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
502: if (flg1) {
503: MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
504: PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
505: }
506: flg1 = PETSC_FALSE;
507: PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
508: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
509: flg1 = PETSC_FALSE;
510: PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
511: if (flg1) {
512: MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
513: }
514: /* experimental */
515: flg1 = PETSC_FALSE;
516: PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
517: if (flg1) {
518: MPI_Errhandler eh;
520: MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
521: MPI_Comm_set_errhandler(comm,eh);
522: MPI_Errhandler_free(&eh);
523: }
524: flg1 = PETSC_FALSE;
525: PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
526: if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
528: /*
529: Setup debugger information
530: */
531: PetscSetDefaultDebugger();
532: PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
533: if (flg1) {
534: MPI_Errhandler err_handler;
536: PetscSetDebuggerFromString(string);
537: MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
538: MPI_Comm_set_errhandler(comm,err_handler);
539: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
540: }
541: PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
542: if (flg1) { PetscSetDebugTerminal(string); }
543: PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
544: PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
545: if (flg1 || flg2) {
546: PetscMPIInt size;
547: PetscInt lsize,*ranks;
548: MPI_Errhandler err_handler;
549: /*
550: we have to make sure that all processors have opened
551: connections to all other processors, otherwise once the
552: debugger has stated it is likely to receive a SIGUSR1
553: and kill the program.
554: */
555: MPI_Comm_size(comm,&size);
556: if (size > 2) {
557: PetscMPIInt dummy = 0;
558: MPI_Status status;
559: for (i=0; i<size; i++) {
560: if (rank != i) {
561: MPI_Send(&dummy,1,MPI_INT,i,109,comm);
562: }
563: }
564: for (i=0; i<size; i++) {
565: if (rank != i) {
566: MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
567: }
568: }
569: }
570: /* check if this processor node should be in debugger */
571: PetscMalloc1(size,&ranks);
572: lsize = size;
573: /* Deprecated in 3.14 */
574: PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);
575: if (flag) {
576: const char * const quietopt="-options_suppress_deprecated_warnings";
577: char msg[4096];
578: PetscBool quiet = PETSC_FALSE;
580: PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);
581: if (!quiet) {
582: PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");
583: PetscStrcat(msg,"-debugger_nodes");
584: PetscStrcat(msg," is deprecated as of version ");
585: PetscStrcat(msg,"3.14");
586: PetscStrcat(msg," and will be removed in a future release.");
587: PetscStrcat(msg," Please use the option ");
588: PetscStrcat(msg,"-debugger_ranks");
589: PetscStrcat(msg," instead.");
590: PetscStrcat(msg," (Silence this warning with ");
591: PetscStrcat(msg,quietopt);
592: PetscStrcat(msg,")\n");
593: PetscPrintf(comm,msg);
594: }
595: } else {
596: lsize = size;
597: PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);
598: }
599: if (flag) {
600: for (i=0; i<lsize; i++) {
601: if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
602: }
603: }
604: if (!flag) {
605: PetscSetDebuggerFromString(string);
606: PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
607: if (flg1) {
608: PetscAttachDebugger();
609: } else {
610: PetscStopForDebugger();
611: }
612: MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
613: MPI_Comm_set_errhandler(comm,err_handler);
614: } else {
615: PetscWaitOnError();
616: }
617: PetscFree(ranks);
618: }
620: PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
621: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
623: /*
624: Setup profiling and logging
625: */
626: #if defined(PETSC_USE_INFO)
627: {
628: PetscInfoSetFromOptions(NULL);
629: }
630: #endif
631: PetscDetermineInitialFPTrap();
632: flg1 = PETSC_FALSE;
633: PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
634: if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
635: PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
637: #if defined(PETSC_USE_LOG)
638: mname[0] = 0;
639: PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
640: if (flg1) {
641: if (mname[0]) {
642: PetscOpenHistoryFile(mname,&petsc_history);
643: } else {
644: PetscOpenHistoryFile(NULL,&petsc_history);
645: }
646: }
648: PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);
650: #if defined(PETSC_HAVE_MPE)
651: flg1 = PETSC_FALSE;
652: PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
653: if (flg1) {PetscLogMPEBegin();}
654: #endif
655: flg1 = PETSC_FALSE;
656: flg3 = PETSC_FALSE;
657: PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
658: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
659: if (flg1) { PetscLogAllBegin(); }
660: else if (flg3) { PetscLogDefaultBegin();}
662: PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
663: if (flg1) {
664: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
665: FILE *file;
666: if (mname[0]) {
667: PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
668: PetscFixFilename(name,fname);
669: file = fopen(fname,"w");
670: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
671: } else file = PETSC_STDOUT;
672: PetscLogTraceBegin(file);
673: }
675: PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
676: if (flg4) {
677: if (format == PETSC_VIEWER_ASCII_XML) {
678: PetscLogNestedBegin();
679: } else {
680: PetscLogDefaultBegin();
681: }
682: }
683: if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
684: PetscReal threshold = PetscRealConstant(0.01);
685: PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
686: if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
687: }
688: #endif
690: PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
691: PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
692: /*
693: If collecting logging information, by default, wait for device to complete its operations
694: before returning to the CPU in order to get accurate timings of each event
695: */
696: PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
697: if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}
699: #if defined(PETSC_HAVE_CUDA)
700: PetscOptionsCheckCUDA(logView);
701: #endif
703: #if defined(PETSC_HAVE_HIP)
704: PetscOptionsCheckHIP(logView);
705: #endif
707: /*
708: Print basic help message
709: */
710: if (hasHelp) {
711: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
712: (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
713: (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
714: (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
715: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
716: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
717: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
718: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
719: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
720: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
721: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
722: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
723: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
724: (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");
725: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
726: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
727: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
728: (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
729: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
730: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
731: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
732: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
733: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
734: (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
735: (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
736: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
737: (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
738: (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
739: (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
740: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
741: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
742: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
743: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
744: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
745: (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
746: #if defined(PETSC_USE_LOG)
747: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
748: (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
749: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
750: (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
751: #if defined(PETSC_HAVE_MPE)
752: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
753: #endif
754: #endif
755: #if defined(PETSC_USE_INFO)
756: (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
757: #endif
758: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
759: (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
760: (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
761: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
762: }
764: #if defined(PETSC_HAVE_POPEN)
765: {
766: char machine[128];
767: PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
768: if (flg1) {
769: PetscPOpenSetMachine(machine);
770: }
771: }
772: #endif
774: PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
775: if (flg1) {
776: PetscSleep(si);
777: }
779: #if defined(PETSC_HAVE_VIENNACL)
780: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
781: if (!flg3) {
782: PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
783: }
784: PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
785: PetscViennaCLSynchronize = flg3;
786: PetscViennaCLInit();
787: #endif
789: /*
790: Creates the logging data structures; this is enabled even if logging is not turned on
791: This is the last thing we do before returning to the user code to prevent having the
792: logging numbers contaminated by any startup time associated with MPI and the GPUs
793: */
794: #if defined(PETSC_USE_LOG)
795: PetscLogInitialize();
796: #endif
798: return(0);
799: }