Actual source code: init.c
petsc-3.13.6 2020-09-29
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot be known
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h>
10: #include <petsc/private/petscimpl.h>
11: #include <petscvalgrind.h>
12: #include <petscviewer.h>
13: #if defined(PETSC_USE_LOG)
14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
15: #endif
17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
18: #include <sys/sysinfo.h>
19: #endif
20: #if defined(PETSC_HAVE_UNISTD_H)
21: #include <unistd.h>
22: #endif
23: #if defined(PETSC_HAVE_CUDA)
24: #include <cuda_runtime.h>
25: #include <petsccublas.h>
26: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
27: #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
28: #endif
29: #endif
31: #if defined(PETSC_HAVE_VIENNACL)
32: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
33: #endif
35: /* ------------------------Nasty global variables -------------------------------*/
36: /*
37: Indicates if PETSc started up MPI, or it was
38: already started before PETSc was initialized.
39: */
40: PetscBool PetscBeganMPI = PETSC_FALSE;
41: PetscBool PetscInitializeCalled = PETSC_FALSE;
42: PetscBool PetscFinalizeCalled = PETSC_FALSE;
43: PetscBool PetscCUDAInitialized = PETSC_FALSE;
45: PetscMPIInt PetscGlobalRank = -1;
46: PetscMPIInt PetscGlobalSize = -1;
48: PetscBool use_gpu_aware_mpi = PETSC_TRUE;
50: #if defined(PETSC_HAVE_COMPLEX)
51: #if defined(PETSC_COMPLEX_INSTANTIATE)
52: template <> class std::complex<double>; /* instantiate complex template class */
53: #endif
54: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
55: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
56: MPI_Datatype MPIU_C_COMPLEX;
57: #endif
59: /*MC
60: PETSC_i - the imaginary number i
62: Synopsis:
63: #include <petscsys.h>
64: PetscComplex PETSC_i;
66: Level: beginner
68: Note:
69: Complex numbers are automatically available if PETSc located a working complex implementation
71: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
72: M*/
73: PetscComplex PETSC_i;
74: #endif
75: #if defined(PETSC_USE_REAL___FLOAT128)
76: MPI_Datatype MPIU___FLOAT128 = 0;
77: #if defined(PETSC_HAVE_COMPLEX)
78: MPI_Datatype MPIU___COMPLEX128 = 0;
79: #endif
80: #elif defined(PETSC_USE_REAL___FP16)
81: MPI_Datatype MPIU___FP16 = 0;
82: #endif
83: MPI_Datatype MPIU_2SCALAR = 0;
84: #if defined(PETSC_USE_64BIT_INDICES)
85: MPI_Datatype MPIU_2INT = 0;
86: #endif
87: MPI_Datatype MPIU_BOOL;
88: MPI_Datatype MPIU_ENUM;
89: MPI_Datatype MPIU_FORTRANADDR;
90: MPI_Datatype MPIU_SIZE_T;
92: /*
93: Function that is called to display all error messages
94: */
95: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
96: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
97: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
98: /*
99: This is needed to turn on/off GPU synchronization
100: */
101: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
102: PetscBool PetscCUDASynchronize = PETSC_FALSE;
104: /* ------------------------------------------------------------------------------*/
105: /*
106: Optional file where all PETSc output from various prints is saved
107: */
108: PETSC_INTERN FILE *petsc_history;
109: FILE *petsc_history = NULL;
111: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
112: {
114: PetscMPIInt rank,size;
115: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
116: char version[256];
119: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
120: if (!rank) {
121: char arch[10];
122: int err;
124: PetscGetArchType(arch,10);
125: PetscGetDate(date,64);
126: PetscGetVersion(version,256);
127: MPI_Comm_size(PETSC_COMM_WORLD,&size);
128: if (filename) {
129: PetscFixFilename(filename,fname);
130: } else {
131: PetscGetHomeDirectory(pfile,240);
132: PetscStrcat(pfile,"/.petschistory");
133: PetscFixFilename(pfile,fname);
134: }
136: *fd = fopen(fname,"a");
137: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
139: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
140: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
141: PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
142: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
143: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
145: err = fflush(*fd);
146: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
147: }
148: return(0);
149: }
151: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
152: {
154: PetscMPIInt rank;
155: char date[64];
156: int err;
159: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
160: if (!rank) {
161: PetscGetDate(date,64);
162: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
163: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
164: PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
165: err = fflush(*fd);
166: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
167: err = fclose(*fd);
168: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
169: }
170: return(0);
171: }
173: /* ------------------------------------------------------------------------------*/
175: /*
176: This is ugly and probably belongs somewhere else, but I want to
177: be able to put a true MPI abort error handler with command line args.
179: This is so MPI errors in the debugger will leave all the stack
180: frames. The default MP_Abort() cleans up and exits thus providing no useful information
181: in the debugger hence we call abort() instead of MPI_Abort().
182: */
184: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
185: {
187: (*PetscErrorPrintf)("MPI error %d\n",*flag);
188: abort();
189: }
191: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
192: {
196: (*PetscErrorPrintf)("MPI error %d\n",*flag);
197: PetscAttachDebugger();
198: if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
199: }
201: #if defined(PETSC_HAVE_CUDA)
202: /*@C
203: PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device
205: Logically collective
207: Input Parameter:
208: comm - the MPI communicator that will utilize the CUDA devices
210: Options Database:
211: + -cuda_initialize <default yes,no> - do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically
212: when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed
213: . -cuda_view - view information about the CUDA devices
214: . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view
215: . -cuda_set_device <gpu> - integer number of the device
216: - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU
218: Level: beginner
220: Notes:
221: Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins
223: @*/
224: PetscErrorCode PetscCUDAInitialize(MPI_Comm comm)
225: {
226: PetscErrorCode ierr;
227: PetscInt deviceOpt = 0;
228: PetscBool cuda_view_flag = PETSC_FALSE,flg;
229: struct cudaDeviceProp prop;
230: int devCount,device,devicecnt;
231: cudaError_t err = cudaSuccess;
232: PetscMPIInt rank,size;
235: /*
236: If collecting logging information, by default, wait for GPU to complete its operations
237: before returning to the CPU in order to get accurate timings of each event
238: */
239: PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);
240: if (!PetscCUDASynchronize) {
241: PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);
242: }
244: PetscOptionsBegin(comm,NULL,"CUDA options","Sys");
245: PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);
246: device = (int)deviceOpt;
247: PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);
248: PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);
249: PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);
250: PetscOptionsEnd();
251: if (!PetscCUDAInitialized) {
252: MPI_Comm_size(comm,&size);
254: if (size>1 && !flg) {
255: /* check to see if we force multiple ranks to hit the same GPU */
256: /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */
258: /* First get the device count */
259: err = cudaGetDeviceCount(&devCount);
260: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
262: /* next determine the rank and then set the device via a mod */
263: MPI_Comm_rank(comm,&rank);
264: device = rank % devCount;
265: }
266: err = cudaSetDevice(device);
267: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
269: /* set the device flags so that it can map host memory */
270: err = cudaSetDeviceFlags(cudaDeviceMapHost);
271: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
273: PetscCUBLASInitializeHandle();
274: PetscCUSOLVERDnInitializeHandle();
275: PetscCUDAInitialized = PETSC_TRUE;
276: }
277: if (cuda_view_flag) {
278: MPI_Comm_rank(comm,&rank);
279: err = cudaGetDeviceCount(&devCount);
280: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
281: for (devicecnt = 0; devicecnt < devCount; ++devicecnt) {
282: err = cudaGetDeviceProperties(&prop,devicecnt);
283: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
284: PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);
285: }
286: PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);
287: PetscSynchronizedFlush(comm,PETSC_STDOUT);
288: }
289: return(0);
290: }
291: #endif
293: /*@C
294: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
295: wishes a clean exit somewhere deep in the program.
297: Collective on PETSC_COMM_WORLD
299: Options Database Keys are the same as for PetscFinalize()
301: Level: advanced
303: Note:
304: See PetscInitialize() for more general runtime options.
306: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
307: @*/
308: PetscErrorCode PetscEnd(void)
309: {
311: PetscFinalize();
312: exit(0);
313: return 0;
314: }
316: PetscBool PetscOptionsPublish = PETSC_FALSE;
317: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
318: PETSC_INTERN PetscBool petscsetmallocvisited;
319: static char emacsmachinename[256];
321: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
322: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL;
324: /*@C
325: PetscSetHelpVersionFunctions - Sets functions that print help and version information
326: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
327: This routine enables a "higher-level" package that uses PETSc to print its messages first.
329: Input Parameter:
330: + help - the help function (may be NULL)
331: - version - the version function (may be NULL)
333: Level: developer
335: @*/
336: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
337: {
339: PetscExternalHelpFunction = help;
340: PetscExternalVersionFunction = version;
341: return(0);
342: }
344: #if defined(PETSC_USE_LOG)
345: PETSC_INTERN PetscBool PetscObjectsLog;
346: #endif
348: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
349: {
350: if (PetscUnlikely(*err)) {
351: PetscMPIInt len;
352: char errstring[MPI_MAX_ERROR_STRING];
354: MPI_Error_string(*err,errstring,&len);
355: PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
356: }
357: return;
358: }
360: PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void)
361: {
362: char string[64];
363: MPI_Comm comm = PETSC_COMM_WORLD;
364: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
365: PetscErrorCode ierr;
366: PetscReal si;
367: PetscInt intensity;
368: int i;
369: PetscMPIInt rank;
370: char version[256],helpoptions[256];
371: #if defined(PETSC_USE_LOG)
372: char mname[PETSC_MAX_PATH_LEN];
373: PetscViewerFormat format;
374: PetscBool flg4 = PETSC_FALSE;
375: #endif
376: #if defined(PETSC_HAVE_CUDA)
377: PetscBool initCUDA = PETSC_TRUE,mpi_gpu_awareness;
378: #endif
381: MPI_Comm_rank(comm,&rank);
383: #if !defined(PETSC_HAVE_THREADSAFETY)
384: if (!(PETSC_RUNNING_ON_VALGRIND)) {
385: /*
386: Setup the memory management; support for tracing malloc() usage
387: */
388: PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;
390: #if defined(PETSC_USE_DEBUG)
391: mdebug = PETSC_TRUE;
392: initializenan = PETSC_TRUE;
393: PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
394: #else
395: /* don't warn about unused option */
396: PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
397: flg1 = PETSC_FALSE;
398: #endif
399: PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
400: if (flg1 || flg2) {
401: mdebug = PETSC_TRUE;
402: eachcall = PETSC_TRUE;
403: initializenan = PETSC_TRUE;
404: } else if (flg3 && !flg2) {
405: mdebug = PETSC_FALSE;
406: eachcall = PETSC_FALSE;
407: initializenan = PETSC_FALSE;
408: }
410: PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
411: if (mlog) {
412: mdebug = PETSC_TRUE;
413: }
414: /* the next line is deprecated */
415: PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
416: PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
417: PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
418: if (mdebug) {
419: PetscMallocSetDebug(eachcall,initializenan);
420: }
421: if (mlog) {
422: PetscReal logthreshold = 0;
423: PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
424: PetscMallocViewSet(logthreshold);
425: }
426: #if defined(PETSC_USE_LOG)
427: PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
428: #endif
429: }
431: PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
432: if (flg2) {PetscMallocSetCoalesce(flg1);}
433: flg1 = PETSC_FALSE;
434: PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
435: /* ignore this option if malloc is already set */
436: if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}
438: flg1 = PETSC_FALSE;
439: PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
440: if (!flg1) {
441: flg1 = PETSC_FALSE;
442: PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
443: }
444: if (flg1) {
445: PetscMemorySetGetMaximumUsage();
446: }
447: #endif
449: #if defined(PETSC_USE_LOG)
450: PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
451: #endif
453: /*
454: Set the display variable for graphics
455: */
456: PetscSetDisplay();
458: /*
459: Print the PETSc version information
460: */
461: PetscOptionsHasName(NULL,NULL,"-v",&flg1);
462: PetscOptionsHasName(NULL,NULL,"-version",&flg2);
463: PetscOptionsHasHelp(NULL,&flg3);
464: if (flg1 || flg2 || flg3) {
466: /*
467: Print "higher-level" package version message
468: */
469: if (PetscExternalVersionFunction) {
470: (*PetscExternalVersionFunction)(comm);
471: }
473: PetscGetVersion(version,256);
474: (*PetscHelpPrintf)(comm,"%s\n",version);
475: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
476: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
477: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
478: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
479: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
480: (*PetscHelpPrintf)(comm,"----------------------------------------\n");
481: }
483: /*
484: Print "higher-level" package help message
485: */
486: if (flg3) {
487: if (PetscExternalHelpFunction) {
488: (*PetscExternalHelpFunction)(comm);
489: }
490: }
492: PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
493: if (flg1) {
494: PetscStrcmp(helpoptions,"intro",&flg2);
495: if (flg2) {
496: PetscOptionsDestroyDefault();
497: PetscFreeMPIResources();
498: MPI_Finalize();
499: exit(0);
500: }
501: }
503: /*
504: Setup the error handling
505: */
506: flg1 = PETSC_FALSE;
507: PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
508: if (flg1) {
509: MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
510: PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
511: }
512: flg1 = PETSC_FALSE;
513: PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
514: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
515: flg1 = PETSC_FALSE;
516: PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
517: if (flg1) {
518: MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
519: }
520: /* experimental */
521: flg1 = PETSC_FALSE;
522: PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
523: if (flg1) {
524: MPI_Errhandler eh;
526: MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
527: MPI_Comm_set_errhandler(comm,eh);
528: MPI_Errhandler_free(&eh);
529: }
530: flg1 = PETSC_FALSE;
531: PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
532: if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
533: flg1 = PETSC_FALSE;
534: PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
535: if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
536: PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
539: /*
540: Setup debugger information
541: */
542: PetscSetDefaultDebugger();
543: PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
544: if (flg1) {
545: MPI_Errhandler err_handler;
547: PetscSetDebuggerFromString(string);
548: MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
549: MPI_Comm_set_errhandler(comm,err_handler);
550: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
551: }
552: PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
553: if (flg1) { PetscSetDebugTerminal(string); }
554: PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
555: PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
556: if (flg1 || flg2) {
557: PetscMPIInt size;
558: PetscInt lsize,*nodes;
559: MPI_Errhandler err_handler;
560: /*
561: we have to make sure that all processors have opened
562: connections to all other processors, otherwise once the
563: debugger has stated it is likely to receive a SIGUSR1
564: and kill the program.
565: */
566: MPI_Comm_size(comm,&size);
567: if (size > 2) {
568: PetscMPIInt dummy = 0;
569: MPI_Status status;
570: for (i=0; i<size; i++) {
571: if (rank != i) {
572: MPI_Send(&dummy,1,MPI_INT,i,109,comm);
573: }
574: }
575: for (i=0; i<size; i++) {
576: if (rank != i) {
577: MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
578: }
579: }
580: }
581: /* check if this processor node should be in debugger */
582: PetscMalloc1(size,&nodes);
583: lsize = size;
584: PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
585: if (flag) {
586: for (i=0; i<lsize; i++) {
587: if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
588: }
589: }
590: if (!flag) {
591: PetscSetDebuggerFromString(string);
592: PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
593: if (flg1) {
594: PetscAttachDebugger();
595: } else {
596: PetscStopForDebugger();
597: }
598: MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
599: MPI_Comm_set_errhandler(comm,err_handler);
600: }
601: PetscFree(nodes);
602: }
604: PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
605: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
607: /*
608: Setup profiling and logging
609: */
610: #if defined(PETSC_USE_INFO)
611: {
612: PetscInfoSetFromOptions(NULL);
613: }
614: #endif
615: #if defined(PETSC_USE_LOG)
616: mname[0] = 0;
617: PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
618: if (flg1) {
619: if (mname[0]) {
620: PetscOpenHistoryFile(mname,&petsc_history);
621: } else {
622: PetscOpenHistoryFile(NULL,&petsc_history);
623: }
624: }
626: PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);
628: #if defined(PETSC_HAVE_MPE)
629: flg1 = PETSC_FALSE;
630: PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
631: if (flg1) {PetscLogMPEBegin();}
632: #endif
633: flg1 = PETSC_FALSE;
634: flg3 = PETSC_FALSE;
635: PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
636: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
637: if (flg1) { PetscLogAllBegin(); }
638: else if (flg3) { PetscLogDefaultBegin();}
640: PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
641: if (flg1) {
642: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
643: FILE *file;
644: if (mname[0]) {
645: PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
646: PetscFixFilename(name,fname);
647: file = fopen(fname,"w");
648: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
649: } else file = PETSC_STDOUT;
650: PetscLogTraceBegin(file);
651: }
653: PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
654: if (flg4) {
655: if (format == PETSC_VIEWER_ASCII_XML) {
656: PetscLogNestedBegin();
657: } else {
658: PetscLogDefaultBegin();
659: }
660: }
661: if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
662: PetscReal threshold = PetscRealConstant(0.01);
663: PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
664: if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
665: }
666: #endif
668: PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
670: #if defined(PETSC_HAVE_CUDA)
671: PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");
672: PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);
673: PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);
674: PetscOptionsEnd();
675: if (initCUDA) {PetscCUDAInitialize(PETSC_COMM_WORLD);}
676: if (use_gpu_aware_mpi) {
677: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
678: /* Trust OpenMPI's compile time cuda query interface */
679: mpi_gpu_awareness = PETSC_TRUE;
680: #else
681: /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults.
682: Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT.
683: */
685: #endif
686: if (!mpi_gpu_awareness) {
687: (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n");
688: (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n");
689: (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n");
690: (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n");
691: (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n");
692: (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n");
693: PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB);
694: }
695: }
696: #endif
698: /*
699: Print basic help message
700: */
701: PetscOptionsHasHelp(NULL,&flg1);
702: if (flg1) {
703: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
704: (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
705: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
706: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
707: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
708: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
709: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
710: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
711: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
712: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
713: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
714: (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
715: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
716: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
717: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
718: (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
719: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
720: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
721: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
722: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
723: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
724: (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
725: (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
726: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
727: (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
728: (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
729: (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
730: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
731: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
732: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
733: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
734: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
735: (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
736: #if defined(PETSC_USE_LOG)
737: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
738: (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
739: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
740: (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
741: #if defined(PETSC_HAVE_MPE)
742: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
743: #endif
744: #endif
745: #if defined(PETSC_USE_INFO)
746: (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
747: #endif
748: (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
749: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
750: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
751: }
753: #if defined(PETSC_HAVE_POPEN)
754: {
755: char machine[128];
756: PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
757: if (flg1) {
758: PetscPOpenSetMachine(machine);
759: }
760: }
761: #endif
763: PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
764: if (flg1) {
765: PetscSleep(si);
766: }
768: #if defined(PETSC_HAVE_VIENNACL)
769: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
770: if (!flg3) {
771: PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
772: }
773: PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
774: PetscViennaCLSynchronize = flg3;
775: PetscViennaCLInit();
776: #endif
778: /*
779: Creates the logging data structures; this is enabled even if logging is not turned on
780: This is the last thing we do before returning to the user code to prevent having the
781: logging numbers contaminated by any startup time associated with MPI and the GPUs
782: */
783: #if defined(PETSC_USE_LOG)
784: PetscLogInitialize();
785: #endif
787: return(0);
788: }