Actual source code: init.c

petsc-3.14.6 2021-03-30
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot be known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>
 10: #include <petsc/private/petscimpl.h>
 11: #include <petscvalgrind.h>
 12: #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif

 24: #if defined(PETSC_HAVE_CUDA)
 25:   #include <cuda_runtime.h>
 26: #include <petsccublas.h>
 27: #endif

 29: #if defined(PETSC_HAVE_HIP)
 30:   #include <hip/hip_runtime.h>
 31: #endif

 33: #if defined(PETSC_HAVE_DEVICE)
 34:   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
 35:     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
 36:   #endif
 37: #endif

 39: #if defined(PETSC_HAVE_VIENNACL)
 40: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 41: #endif


 44: /* ------------------------Nasty global variables -------------------------------*/
 45: /*
 46:      Indicates if PETSc started up MPI, or it was
 47:    already started before PETSc was initialized.
 48: */
 49: PetscBool   PetscBeganMPI                 = PETSC_FALSE;
 50: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
 51: PetscBool   PetscInitializeCalled         = PETSC_FALSE;
 52: PetscBool   PetscFinalizeCalled           = PETSC_FALSE;

 54: PetscMPIInt PetscGlobalRank               = -1;
 55: PetscMPIInt PetscGlobalSize               = -1;

 57: #if defined(PETSC_HAVE_KOKKOS)
 58: PetscBool   PetscBeganKokkos              = PETSC_FALSE;
 59: #endif

 61: PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;
 62: PetscBool   PetscCreatedGpuObjects        = PETSC_FALSE;

 64: #if defined(PETSC_HAVE_COMPLEX)
 65: #if defined(PETSC_COMPLEX_INSTANTIATE)
 66: template <> class std::complex<double>; /* instantiate complex template class */
 67: #endif
 68: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 69: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 70: MPI_Datatype MPIU_C_COMPLEX;
 71: #endif

 73: /*MC
 74:    PETSC_i - the imaginary number i

 76:    Synopsis:
 77: #include <petscsys.h>
 78:    PetscComplex PETSC_i;

 80:    Level: beginner

 82:    Note:
 83:    Complex numbers are automatically available if PETSc located a working complex implementation

 85: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 86: M*/
 87: PetscComplex PETSC_i;
 88: #endif
 89: #if defined(PETSC_USE_REAL___FLOAT128)
 90: MPI_Datatype MPIU___FLOAT128 = 0;
 91: #if defined(PETSC_HAVE_COMPLEX)
 92: MPI_Datatype MPIU___COMPLEX128 = 0;
 93: #endif
 94: #elif defined(PETSC_USE_REAL___FP16)
 95: MPI_Datatype MPIU___FP16 = 0;
 96: #endif
 97: MPI_Datatype MPIU_2SCALAR = 0;
 98: #if defined(PETSC_USE_64BIT_INDICES)
 99: MPI_Datatype MPIU_2INT = 0;
100: #endif
101: MPI_Datatype MPIU_BOOL;
102: MPI_Datatype MPIU_ENUM;
103: MPI_Datatype MPIU_FORTRANADDR;
104: MPI_Datatype MPIU_SIZE_T;

106: /*
107:        Function that is called to display all error messages
108: */
109: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
110: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
111: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
112: /*
113:   This is needed to turn on/off GPU synchronization
114: */
115: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;

117: /* ------------------------------------------------------------------------------*/
118: /*
119:    Optional file where all PETSc output from various prints is saved
120: */
121: PETSC_INTERN FILE *petsc_history;
122: FILE *petsc_history = NULL;

124: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
125: {
127:   PetscMPIInt    rank,size;
128:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
129:   char           version[256];

132:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
133:   if (!rank) {
134:     char        arch[10];
135:     int         err;

137:     PetscGetArchType(arch,10);
138:     PetscGetDate(date,64);
139:     PetscGetVersion(version,256);
140:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
141:     if (filename) {
142:       PetscFixFilename(filename,fname);
143:     } else {
144:       PetscGetHomeDirectory(pfile,sizeof(pfile));
145:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
146:       PetscFixFilename(pfile,fname);
147:     }

149:     *fd = fopen(fname,"a");
150:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
153:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
154:     PetscGetProgramName(pname,sizeof(pname));
155:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
156:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

158:     err = fflush(*fd);
159:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160:   }
161:   return(0);
162: }

164: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
165: {
167:   PetscMPIInt    rank;
168:   char           date[64];
169:   int            err;

172:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
173:   if (!rank) {
174:     PetscGetDate(date,64);
175:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
176:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
177:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
178:     err  = fflush(*fd);
179:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
180:     err = fclose(*fd);
181:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
182:   }
183:   return(0);
184: }

186: /* ------------------------------------------------------------------------------*/

188: /*
189:    This is ugly and probably belongs somewhere else, but I want to
190:   be able to put a true MPI abort error handler with command line args.

192:     This is so MPI errors in the debugger will leave all the stack
193:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
194:   in the debugger hence we call abort() instead of MPI_Abort().
195: */

197: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
198: {
200:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
201:   abort();
202: }

204: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
205: {

209:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
210:   PetscAttachDebugger();
211:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
212: }

214: /*@C
215:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
216:      wishes a clean exit somewhere deep in the program.

218:    Collective on PETSC_COMM_WORLD

220:    Options Database Keys are the same as for PetscFinalize()

222:    Level: advanced

224:    Note:
225:    See PetscInitialize() for more general runtime options.

227: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
228: @*/
229: PetscErrorCode  PetscEnd(void)
230: {
232:   PetscFinalize();
233:   exit(0);
234:   return 0;
235: }

237: PetscBool PetscOptionsPublish = PETSC_FALSE;
238: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
239: PETSC_INTERN PetscBool      petscsetmallocvisited;
240: static       char           emacsmachinename[256];

242: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
243: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

245: /*@C
246:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
247:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
248:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

250:    Input Parameter:
251: +  help - the help function (may be NULL)
252: -  version - the version function (may be NULL)

254:    Level: developer

256: @*/
257: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
258: {
260:   PetscExternalHelpFunction    = help;
261:   PetscExternalVersionFunction = version;
262:   return(0);
263: }

265: #if defined(PETSC_USE_LOG)
266: PETSC_INTERN PetscBool   PetscObjectsLog;
267: #endif

269: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
270: {
271:   if (PetscUnlikely(*err)) {
272:     PETSC_UNUSED PetscMPIInt len;
273:     char                     errstring[MPI_MAX_ERROR_STRING];

275:     MPI_Error_string(*err,errstring,&len);
276:     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
277:   }
278:   return;
279: }

281: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
282:    Use the following macros to define CUDA/HIP initialization related vars/routines.
283:  */
284: #if defined(PETSC_HAVE_CUDA)
285:   typedef cudaError_t                             cupmError_t;
286:   typedef struct cudaDeviceProp                   cupmDeviceProp;
287:   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
288:   #define cupmGetDevice(x)                        cudaGetDevice(x)
289:   #define cupmSetDevice(x)                        cudaSetDevice(x)
290:   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
291:   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
292:   #define cupmGetLastError()                      cudaGetLastError()
293:   #define cupmDeviceMapHost                       cudaDeviceMapHost
294:   #define cupmSuccess                             cudaSuccess
295:   #define cupmErrorMemoryAllocation               cudaErrorMemoryAllocation
296:   #define cupmErrorLaunchOutOfResources           cudaErrorLaunchOutOfResources
297:   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
298:   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
299:   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
300:   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
301:   #define PetscCUPMInitialize                     PetscCUDAInitialize
302:   #define PetscCUPMInitialized                    PetscCUDAInitialized
303:   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
304:   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
305:   #define PetscCUPMSynchronize                    PetscCUDASynchronize
306:   #define PetscNotUseCUPM                         PetscNotUseCUDA
307:   #define cupmOptionsStr                          "CUDA options"
308:   #define cupmSetDeviceStr                        "-cuda_device"
309:   #define cupmViewStr                             "-cuda_view"
310:   #define cupmSynchronizeStr                      "-cuda_synchronize"
311:   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
312:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
313:   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
314:   #include "cupminit.inc"
315: #endif

317: #if defined(PETSC_HAVE_HIP)
318:   typedef hipError_t                              cupmError_t;
319:   typedef hipDeviceProp_t                         cupmDeviceProp;
320:   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
321:   #define cupmGetDevice(x)                        hipGetDevice(x)
322:   #define cupmSetDevice(x)                        hipSetDevice(x)
323:   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
324:   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
325:   #define cupmGetLastError()                      hipGetLastError()
326:   #define cupmDeviceMapHost                       hipDeviceMapHost
327:   #define cupmSuccess                             hipSuccess
328:   #define cupmErrorMemoryAllocation               hipErrorMemoryAllocation
329:   #define cupmErrorLaunchOutOfResources           hipErrorLaunchOutOfResources
330:   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
331:   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
332:   #define PetscCUPMBLASInitializeHandle()         0
333:   #define PetscCUPMSOLVERDnInitializeHandle()     0
334:   #define PetscCUPMInitialize                     PetscHIPInitialize
335:   #define PetscCUPMInitialized                    PetscHIPInitialized
336:   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
337:   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
338:   #define PetscCUPMSynchronize                    PetscHIPSynchronize
339:   #define PetscNotUseCUPM                         PetscNotUseHIP
340:   #define cupmOptionsStr                          "HIP options"
341:   #define cupmSetDeviceStr                        "-hip_device"
342:   #define cupmViewStr                             "-hip_view"
343:   #define cupmSynchronizeStr                      "-hip_synchronize"
344:   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
345:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
346:   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
347:   #include "cupminit.inc"
348: #endif

350: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
351: {
352:   char              string[64];
353:   MPI_Comm          comm = PETSC_COMM_WORLD;
354:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
355:   PetscErrorCode    ierr;
356:   PetscReal         si;
357:   PetscInt          intensity;
358:   int               i;
359:   PetscMPIInt       rank;
360:   char              version[256];
361: #if defined(PETSC_USE_LOG)
362:   char              mname[PETSC_MAX_PATH_LEN];
363:   PetscViewerFormat format;
364:   PetscBool         flg4 = PETSC_FALSE;
365: #endif

368:   MPI_Comm_rank(comm,&rank);

370: #if !defined(PETSC_HAVE_THREADSAFETY)
371:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
372:     /*
373:       Setup the memory management; support for tracing malloc() usage
374:     */
375:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

377:     if (PetscDefined(USE_DEBUG)) {
378:       mdebug        = PETSC_TRUE;
379:       initializenan = PETSC_TRUE;
380:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
381:     } else {
382:       /* don't warn about unused option */
383:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
384:       flg1 = PETSC_FALSE;
385:     }
386:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
387:     if (flg1 || flg2) {
388:       mdebug        = PETSC_TRUE;
389:       eachcall      = PETSC_TRUE;
390:       initializenan = PETSC_TRUE;
391:     } else if (flg3 && !flg2) {
392:       mdebug        = PETSC_FALSE;
393:       eachcall      = PETSC_FALSE;
394:       initializenan = PETSC_FALSE;
395:     }

397:     PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);
398:     if (flg2) {PetscMallocLogRequestedSizeSet(flg1);}

400:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
401:     if (mlog) {
402:       mdebug = PETSC_TRUE;
403:     }
404:     /* the next line is deprecated */
405:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
406:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
407:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
408:     if (mdebug) {
409:       PetscMallocSetDebug(eachcall,initializenan);
410:     }
411:     if (mlog) {
412:       PetscReal logthreshold = 0;
413:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
414:       PetscMallocViewSet(logthreshold);
415:     }
416: #if defined(PETSC_USE_LOG)
417:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
418: #endif
419:   }

421:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
422:   if (flg2) {PetscMallocSetCoalesce(flg1);}
423:   flg1 = PETSC_FALSE;
424:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
425:   /* ignore this option if malloc is already set */
426:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

428:   flg1 = PETSC_FALSE;
429:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
430:   if (!flg1) {
431:     flg1 = PETSC_FALSE;
432:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
433:   }
434:   if (flg1) {
435:     PetscMemorySetGetMaximumUsage();
436:   }
437: #endif

439: #if defined(PETSC_USE_LOG)
440:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
441: #endif

443:   /*
444:       Set the display variable for graphics
445:   */
446:   PetscSetDisplay();

448:   /*
449:      Print main application help message
450:   */
451:   PetscOptionsHasHelp(NULL,&hasHelp);
452:   if (help && hasHelp) {
453:     PetscPrintf(comm,help);
454:     PetscPrintf(comm,"----------------------------------------\n");
455:   }

457:   /*
458:       Print the PETSc version information
459:   */
460:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
461:   if (flg1 || hasHelp) {
462:     /*
463:        Print "higher-level" package version message
464:     */
465:     if (PetscExternalVersionFunction) {
466:       (*PetscExternalVersionFunction)(comm);
467:     }

469:     PetscGetVersion(version,256);
470:     (*PetscHelpPrintf)(comm,"%s\n",version);
471:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
472:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
473:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
474:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
475:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
476:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
477:   }

479:   /*
480:        Print "higher-level" package help message
481:   */
482:   if (hasHelp) {
483:     PetscBool hasHelpIntro;

485:     if (PetscExternalHelpFunction) {
486:       (*PetscExternalHelpFunction)(comm);
487:     }
488:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
489:     if (hasHelpIntro) {
490:       PetscOptionsDestroyDefault();
491:       PetscFreeMPIResources();
492:       MPI_Finalize();
493:       exit(0);
494:     }
495:   }

497:   /*
498:       Setup the error handling
499:   */
500:   flg1 = PETSC_FALSE;
501:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
502:   if (flg1) {
503:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
504:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
505:   }
506:   flg1 = PETSC_FALSE;
507:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
508:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
509:   flg1 = PETSC_FALSE;
510:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
511:   if (flg1) {
512:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
513:   }
514:   /* experimental */
515:   flg1 = PETSC_FALSE;
516:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
517:   if (flg1) {
518:     MPI_Errhandler eh;

520:     MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
521:     MPI_Comm_set_errhandler(comm,eh);
522:     MPI_Errhandler_free(&eh);
523:   }
524:   flg1 = PETSC_FALSE;
525:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
526:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

528:   /*
529:       Setup debugger information
530:   */
531:   PetscSetDefaultDebugger();
532:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
533:   if (flg1) {
534:     MPI_Errhandler err_handler;

536:     PetscSetDebuggerFromString(string);
537:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
538:     MPI_Comm_set_errhandler(comm,err_handler);
539:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
540:   }
541:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
542:   if (flg1) { PetscSetDebugTerminal(string); }
543:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
544:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
545:   if (flg1 || flg2) {
546:     PetscMPIInt    size;
547:     PetscInt       lsize,*ranks;
548:     MPI_Errhandler err_handler;
549:     /*
550:        we have to make sure that all processors have opened
551:        connections to all other processors, otherwise once the
552:        debugger has stated it is likely to receive a SIGUSR1
553:        and kill the program.
554:     */
555:     MPI_Comm_size(comm,&size);
556:     if (size > 2) {
557:       PetscMPIInt dummy = 0;
558:       MPI_Status  status;
559:       for (i=0; i<size; i++) {
560:         if (rank != i) {
561:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
562:         }
563:       }
564:       for (i=0; i<size; i++) {
565:         if (rank != i) {
566:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
567:         }
568:       }
569:     }
570:     /* check if this processor node should be in debugger */
571:     PetscMalloc1(size,&ranks);
572:     lsize = size;
573:     /* Deprecated in 3.14 */
574:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);
575:     if (flag) {
576:       const char * const quietopt="-options_suppress_deprecated_warnings";
577:       char               msg[4096];
578:       PetscBool          quiet = PETSC_FALSE;

580:       PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);
581:       if (!quiet) {
582:         PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");
583:         PetscStrcat(msg,"-debugger_nodes");
584:         PetscStrcat(msg," is deprecated as of version ");
585:         PetscStrcat(msg,"3.14");
586:         PetscStrcat(msg," and will be removed in a future release.");
587:         PetscStrcat(msg," Please use the option ");
588:         PetscStrcat(msg,"-debugger_ranks");
589:         PetscStrcat(msg," instead.");
590:         PetscStrcat(msg," (Silence this warning with ");
591:         PetscStrcat(msg,quietopt);
592:         PetscStrcat(msg,")\n");
593:         PetscPrintf(comm,msg);
594:       }
595:     } else {
596:       lsize = size;
597:       PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);
598:     }
599:     if (flag) {
600:       for (i=0; i<lsize; i++) {
601:         if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
602:       }
603:     }
604:     if (!flag) {
605:       PetscSetDebuggerFromString(string);
606:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
607:       if (flg1) {
608:         PetscAttachDebugger();
609:       } else {
610:         PetscStopForDebugger();
611:       }
612:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
613:       MPI_Comm_set_errhandler(comm,err_handler);
614:     } else {
615:       PetscWaitOnError();
616:     }
617:     PetscFree(ranks);
618:   }

620:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
621:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

623:   /*
624:         Setup profiling and logging
625:   */
626: #if defined(PETSC_USE_INFO)
627:   {
628:     PetscInfoSetFromOptions(NULL);
629:   }
630: #endif
631:   PetscDetermineInitialFPTrap();
632:   flg1 = PETSC_FALSE;
633:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
634:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
635:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
637: #if defined(PETSC_USE_LOG)
638:   mname[0] = 0;
639:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
640:   if (flg1) {
641:     if (mname[0]) {
642:       PetscOpenHistoryFile(mname,&petsc_history);
643:     } else {
644:       PetscOpenHistoryFile(NULL,&petsc_history);
645:     }
646:   }

648:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

650: #if defined(PETSC_HAVE_MPE)
651:   flg1 = PETSC_FALSE;
652:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
653:   if (flg1) {PetscLogMPEBegin();}
654: #endif
655:   flg1 = PETSC_FALSE;
656:   flg3 = PETSC_FALSE;
657:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
658:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
659:   if (flg1)                      { PetscLogAllBegin(); }
660:   else if (flg3)                 { PetscLogDefaultBegin();}

662:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
663:   if (flg1) {
664:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
665:     FILE *file;
666:     if (mname[0]) {
667:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
668:       PetscFixFilename(name,fname);
669:       file = fopen(fname,"w");
670:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
671:     } else file = PETSC_STDOUT;
672:     PetscLogTraceBegin(file);
673:   }

675:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
676:   if (flg4) {
677:     if (format == PETSC_VIEWER_ASCII_XML) {
678:       PetscLogNestedBegin();
679:     } else {
680:       PetscLogDefaultBegin();
681:     }
682:   }
683:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
684:     PetscReal threshold = PetscRealConstant(0.01);
685:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
686:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
687:   }
688: #endif

690:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
691:   PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
692:   /*
693:     If collecting logging information, by default, wait for device to complete its operations
694:     before returning to the CPU in order to get accurate timings of each event
695:   */
696:   PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
697:   if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}

699: #if defined(PETSC_HAVE_CUDA)
700:   PetscOptionsCheckCUDA(logView);
701: #endif

703: #if defined(PETSC_HAVE_HIP)
704:   PetscOptionsCheckHIP(logView);
705: #endif

707:   /*
708:        Print basic help message
709:   */
710:   if (hasHelp) {
711:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
712:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
713:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
714:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
715:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
716:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
717:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
718:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
719:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
720:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
721:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
722:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
723:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
724:     (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");
725:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
726:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
727:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
728:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
729:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
730:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
731:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
732:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
733:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
734:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
735:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
736:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
737:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
738:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
739:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
740:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
741:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
742:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
743:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
744:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
745:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
746: #if defined(PETSC_USE_LOG)
747:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
748:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
749:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
750:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
751: #if defined(PETSC_HAVE_MPE)
752:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
753: #endif
754: #endif
755: #if defined(PETSC_USE_INFO)
756:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
757: #endif
758:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
759:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
760:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
761:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
762:   }

764: #if defined(PETSC_HAVE_POPEN)
765:   {
766:   char machine[128];
767:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
768:   if (flg1) {
769:     PetscPOpenSetMachine(machine);
770:   }
771:   }
772: #endif

774:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
775:   if (flg1) {
776:     PetscSleep(si);
777:   }

779: #if defined(PETSC_HAVE_VIENNACL)
780:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
781:   if (!flg3) {
782:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
783:   }
784:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
785:   PetscViennaCLSynchronize = flg3;
786:   PetscViennaCLInit();
787: #endif

789:   /*
790:      Creates the logging data structures; this is enabled even if logging is not turned on
791:      This is the last thing we do before returning to the user code to prevent having the
792:      logging numbers contaminated by any startup time associated with MPI and the GPUs
793:   */
794: #if defined(PETSC_USE_LOG)
795:   PetscLogInitialize();
796: #endif

798:   return(0);
799: }