Actual source code: init.c

petsc-3.11.4 2019-09-28
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>

 14: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 15: #include <sys/sysinfo.h>
 16: #endif
 17: #if defined(PETSC_HAVE_UNISTD_H)
 18: #include <unistd.h>
 19: #endif
 20: #if defined(PETSC_HAVE_CUDA)
 21: #include <cuda_runtime.h>
 22: #endif

 24: #if defined(PETSC_HAVE_VIENNACL)
 25: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 26: #endif

 28: /* ------------------------Nasty global variables -------------------------------*/
 29: /*
 30:      Indicates if PETSc started up MPI, or it was
 31:    already started before PETSc was initialized.
 32: */
 33: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 34: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 35: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 36: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 38: PetscMPIInt PetscGlobalRank       = -1;
 39: PetscMPIInt PetscGlobalSize       = -1;

 41: #if defined(PETSC_HAVE_COMPLEX)
 42: #if defined(PETSC_COMPLEX_INSTANTIATE)
 43: template <> class std::complex<double>; /* instantiate complex template class */
 44: #endif
 45: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 46: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 47: MPI_Datatype MPIU_C_COMPLEX;
 48: #endif

 50: /*MC
 51:    PETSC_i - the imaginary number i

 53:    Synopsis:
 54:    #include <petscsys.h>
 55:    PetscComplex PETSC_i;

 57:    Level: beginner

 59:    Note:
 60:    Complex numbers are automatically available if PETSc located a working complex implementation

 62: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 63: M*/
 64: PetscComplex PETSC_i;
 65: #endif
 66: #if defined(PETSC_USE_REAL___FLOAT128)
 67: MPI_Datatype MPIU___FLOAT128 = 0;
 68: #if defined(PETSC_HAVE_COMPLEX)
 69: MPI_Datatype MPIU___COMPLEX128 = 0;
 70: #endif
 71: #elif defined(PETSC_USE_REAL___FP16)
 72: MPI_Datatype MPIU___FP16 = 0;
 73: #endif
 74: MPI_Datatype MPIU_2SCALAR = 0;
 75: #if defined(PETSC_USE_64BIT_INDICES)
 76: MPI_Datatype MPIU_2INT = 0;
 77: #endif
 78: MPI_Datatype MPIU_BOOL;
 79: MPI_Datatype MPIU_ENUM;

 81: /*
 82:        Function that is called to display all error messages
 83: */
 84: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 85: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 86: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 87: /*
 88:   This is needed to turn on/off GPU synchronization
 89: */
 90: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
 91: PetscBool PetscCUDASynchronize = PETSC_FALSE;

 93: /* ------------------------------------------------------------------------------*/
 94: /*
 95:    Optional file where all PETSc output from various prints is saved
 96: */
 97: PETSC_INTERN FILE *petsc_history;
 98: FILE *petsc_history = NULL;

100: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
101: {
103:   PetscMPIInt    rank,size;
104:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
105:   char           version[256];

108:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
109:   if (!rank) {
110:     char        arch[10];
111:     int         err;

113:     PetscGetArchType(arch,10);
114:     PetscGetDate(date,64);
115:     PetscGetVersion(version,256);
116:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
117:     if (filename) {
118:       PetscFixFilename(filename,fname);
119:     } else {
120:       PetscGetHomeDirectory(pfile,240);
121:       PetscStrcat(pfile,"/.petschistory");
122:       PetscFixFilename(pfile,fname);
123:     }

125:     *fd = fopen(fname,"a");
126:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

128:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
129:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
130:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
131:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
132:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");

134:     err = fflush(*fd);
135:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
136:   }
137:   return(0);
138: }

140: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
141: {
143:   PetscMPIInt    rank;
144:   char           date[64];
145:   int            err;

148:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
149:   if (!rank) {
150:     PetscGetDate(date,64);
151:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
153:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
154:     err  = fflush(*fd);
155:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
156:     err = fclose(*fd);
157:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
158:   }
159:   return(0);
160: }

162: /* ------------------------------------------------------------------------------*/

164: /*
165:    This is ugly and probably belongs somewhere else, but I want to
166:   be able to put a true MPI abort error handler with command line args.

168:     This is so MPI errors in the debugger will leave all the stack
169:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
170:   in the debugger hence we call abort() instead of MPI_Abort().
171: */

173: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
174: {
176:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
177:   abort();
178: }

180: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
181: {

185:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
186:   PetscAttachDebugger();
187:   if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
188: }

190: /*@C
191:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
192:      wishes a clean exit somewhere deep in the program.

194:    Collective on PETSC_COMM_WORLD

196:    Options Database Keys are the same as for PetscFinalize()

198:    Level: advanced

200:    Note:
201:    See PetscInitialize() for more general runtime options.

203: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
204: @*/
205: PetscErrorCode  PetscEnd(void)
206: {
208:   PetscFinalize();
209:   exit(0);
210:   return 0;
211: }

213: PetscBool PetscOptionsPublish = PETSC_FALSE;
214: PETSC_INTERN PetscErrorCode PetscSetUseTrMalloc_Private(void);
215: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
216: PETSC_INTERN PetscBool      petscsetmallocvisited;
217: static       char           emacsmachinename[256];

219: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
220: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

222: /*@C
223:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
224:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
225:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

227:    Input Parameter:
228: +  help - the help function (may be NULL)
229: -  version - the version function (may be NULL)

231:    Level: developer

233:    Concepts: package help message

235: @*/
236: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
237: {
239:   PetscExternalHelpFunction    = help;
240:   PetscExternalVersionFunction = version;
241:   return(0);
242: }

244: #if defined(PETSC_USE_LOG)
245: PETSC_INTERN PetscBool   PetscObjectsLog;
246: #endif

248: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(void)
249: {
250:   char              string[64],mname[PETSC_MAX_PATH_LEN],*f;
251:   MPI_Comm          comm = PETSC_COMM_WORLD;
252:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
253:   PetscErrorCode    ierr;
254:   PetscReal         si;
255:   PetscInt          intensity;
256:   int               i;
257:   PetscMPIInt       rank;
258:   char              version[256],helpoptions[256];
259: #if !defined(PETSC_HAVE_THREADSAFETY)
260:   PetscReal         logthreshold;
261: #endif
262: #if defined(PETSC_USE_LOG)
263:   PetscViewerFormat format;
264:   PetscBool         flg4 = PETSC_FALSE;
265: #endif

268:   MPI_Comm_rank(comm,&rank);

270: #if !defined(PETSC_HAVE_THREADSAFETY)
271:   /*
272:       Setup the memory management; support for tracing malloc() usage
273:   */
274:   PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
275:   logthreshold = 0.0;
276:   PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
277:   if (flg1) flg3 = PETSC_TRUE;
278: #if defined(PETSC_USE_DEBUG)
279:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
280:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
281:     if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
282:       /* turn off default -malloc if valgrind is being used */
283:       PetscSetUseTrMalloc_Private();
284:     }
285:   }
286: #else
287:   PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
288:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
289:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
290: #endif
291:   if (flg3) {
292:     PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
293:   }
294:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
295:   if (flg2) {PetscMallocSetCoalesce(flg1);}
296:   flg1 = PETSC_FALSE;
297:   PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
298:   if (flg1) {
299:     PetscSetUseTrMalloc_Private();
300:     PetscMallocDebug(PETSC_TRUE);
301:   }
302:   flg1 = PETSC_FALSE;
303:   PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
304: #if defined(PETSC_USE_DEBUG)
305:   if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
306:     PetscSetUseTrMalloc_Private();
307:     PetscMallocSetDumpLog();
308:     PetscMallocDebug(PETSC_TRUE);
309:   }
310: #endif
311:   flg1 = PETSC_FALSE;
312:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
313:   /* ignore this option if malloc is already set */
314:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

316:   flg1 = PETSC_FALSE;
317:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
318:   if (!flg1) {
319:     flg1 = PETSC_FALSE;
320:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
321:   }
322:   if (flg1) {
323:     PetscMemorySetGetMaximumUsage();
324:   }
325: #endif

327: #if defined(PETSC_USE_LOG)
328:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
329: #endif

331:   /*
332:       Set the display variable for graphics
333:   */
334:   PetscSetDisplay();

336:   /*
337:       Print the PETSc version information
338:   */
339:   PetscOptionsHasName(NULL,NULL,"-v",&flg1);
340:   PetscOptionsHasName(NULL,NULL,"-version",&flg2);
341:   PetscOptionsHasHelp(NULL,&flg3);
342:   if (flg1 || flg2 || flg3) {

344:     /*
345:        Print "higher-level" package version message
346:     */
347:     if (PetscExternalVersionFunction) {
348:       (*PetscExternalVersionFunction)(comm);
349:     }

351:     PetscGetVersion(version,256);
352:     (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
353:     (*PetscHelpPrintf)(comm,"%s\n",version);
354:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
355:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
356:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
357:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
358:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
359:     (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
360:   }

362:   /*
363:        Print "higher-level" package help message
364:   */
365:   if (flg3) {
366:     if (PetscExternalHelpFunction) {
367:       (*PetscExternalHelpFunction)(comm);
368:     }
369:   }

371:   PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
372:   if (flg1) {
373:     PetscStrcmp(helpoptions,"intro",&flg2);
374:     if (flg2) {
375:       PetscOptionsDestroyDefault();
376:       PetscFreeMPIResources();
377:       MPI_Finalize();
378:       exit(0);
379:     }
380:   }

382:   /*
383:       Setup the error handling
384:   */
385:   flg1 = PETSC_FALSE;
386:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
387:   if (flg1) {
388:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
389:     PetscPushErrorHandler(PetscAbortErrorHandler,0);
390:   }
391:   flg1 = PETSC_FALSE;
392:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
393:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
394:   flg1 = PETSC_FALSE;
395:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
396:   if (flg1) {
397:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
398:   }
399:   flg1 = PETSC_FALSE;
400:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
401:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
402:   flg1 = PETSC_FALSE;
403:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
404:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
405:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);

408:   /*
409:       Setup debugger information
410:   */
411:   PetscSetDefaultDebugger();
412:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
413:   if (flg1) {
414:     MPI_Errhandler err_handler;

416:     PetscSetDebuggerFromString(string);
417:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
418:     MPI_Comm_set_errhandler(comm,err_handler);
419:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
420:   }
421:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
422:   if (flg1) { PetscSetDebugTerminal(string); }
423:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
424:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
425:   if (flg1 || flg2) {
426:     PetscMPIInt    size;
427:     PetscInt       lsize,*nodes;
428:     MPI_Errhandler err_handler;
429:     /*
430:        we have to make sure that all processors have opened
431:        connections to all other processors, otherwise once the
432:        debugger has stated it is likely to receive a SIGUSR1
433:        and kill the program.
434:     */
435:     MPI_Comm_size(comm,&size);
436:     if (size > 2) {
437:       PetscMPIInt dummy = 0;
438:       MPI_Status  status;
439:       for (i=0; i<size; i++) {
440:         if (rank != i) {
441:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
442:         }
443:       }
444:       for (i=0; i<size; i++) {
445:         if (rank != i) {
446:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
447:         }
448:       }
449:     }
450:     /* check if this processor node should be in debugger */
451:     PetscMalloc1(size,&nodes);
452:     lsize = size;
453:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
454:     if (flag) {
455:       for (i=0; i<lsize; i++) {
456:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
457:       }
458:     }
459:     if (!flag) {
460:       PetscSetDebuggerFromString(string);
461:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
462:       if (flg1) {
463:         PetscAttachDebugger();
464:       } else {
465:         PetscStopForDebugger();
466:       }
467:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
468:       MPI_Comm_set_errhandler(comm,err_handler);
469:     }
470:     PetscFree(nodes);
471:   }

473:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
474:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

476:   /*
477:         Setup profiling and logging
478:   */
479: #if defined(PETSC_USE_INFO)
480:   {
481:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
482:     PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
483:     if (flg1 && logname[0]) {
484:       PetscInfoAllow(PETSC_TRUE,logname);
485:     } else if (flg1) {
486:       PetscInfoAllow(PETSC_TRUE,NULL);
487:     }
488:   }
489: #endif
490: #if defined(PETSC_USE_LOG)
491:   mname[0] = 0;
492:   PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
493:   if (flg1) {
494:     if (mname[0]) {
495:       PetscOpenHistoryFile(mname,&petsc_history);
496:     } else {
497:       PetscOpenHistoryFile(NULL,&petsc_history);
498:     }
499:   }

501:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

503: #if defined(PETSC_HAVE_MPE)
504:   flg1 = PETSC_FALSE;
505:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
506:   if (flg1) {PetscLogMPEBegin();}
507: #endif
508:   flg1 = PETSC_FALSE;
509:   flg3 = PETSC_FALSE;
510:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
511:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
512:   if (flg1)                      { PetscLogAllBegin(); }
513:   else if (flg3)                 { PetscLogDefaultBegin();}

515:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
516:   if (flg1) {
517:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
518:     FILE *file;
519:     if (mname[0]) {
520:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
521:       PetscFixFilename(name,fname);
522:       file = fopen(fname,"w");
523:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
524:     } else file = PETSC_STDOUT;
525:     PetscLogTraceBegin(file);
526:   }

528:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
529:   if (flg4) {
530:     if (format == PETSC_VIEWER_ASCII_XML) {
531:       PetscLogNestedBegin();
532:     } else {
533:       PetscLogDefaultBegin();
534:     }
535:   }
536:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
537:     PetscReal threshold = PetscRealConstant(0.01);
538:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
539:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
540:   }
541: #endif

543:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

545: #if defined(PETSC_HAVE_CUDA)
546:   PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
547:   if (flg1) {
548:     struct cudaDeviceProp prop;
549:     int                   devCount;
550:     PetscInt              device;
551:     cudaError_t           err = cudaSuccess;

553:     err = cudaGetDeviceCount(&devCount);
554:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
555:     for (device = 0; device < devCount; ++device) {
556:       err = cudaGetDeviceProperties(&prop, (int)device);
557:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
558:       PetscPrintf(comm, "CUDA device %D: %s\n", device, prop.name);
559:     }
560:   }
561:   if (!PetscCUDAInitialized) {
562:     PetscMPIInt size;
563:     MPI_Comm_size(comm,&size);
564:     if (size>1) {
565:       int         devCount;
566:       PetscInt    device;
567:       PetscMPIInt rank;
568:       cudaError_t err = cudaSuccess;

570:       /* check to see if we force multiple ranks to hit the same GPU */
571:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
572:       if (flg1) {
573:         err = cudaSetDevice((int)device);
574:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
575:       } else {
576:         /* we're not using the same GPU on multiple MPI threads. So try to allocated different   GPUs to different processes */

578:         /* First get the device count */
579:         err   = cudaGetDeviceCount(&devCount);
580:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));

582:         /* next determine the rank and then set the device via a mod */
583:         MPI_Comm_rank(comm,&rank);
584:         device = rank % devCount;
585:         err    = cudaSetDevice((int)device);
586:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
587:       }

589:       /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
590:        multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
591:        are ok. */
592:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
593:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
594:     } else {
595:       PetscInt    device;
596:       cudaError_t err = cudaSuccess;

598:       /* the code below works for serial GPU simulations */
599:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
600:       if (flg1) {
601:         err = cudaSetDevice((int)device);
602:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
603:       }

605:       /* set the device flags so that it can map host memory ... here, we error check. */
606:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
607:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
608:     }

610:     PetscCUDAInitialized = PETSC_TRUE;
611:   }
612: #endif


615:   /*
616:        Print basic help message
617:   */
618:   PetscOptionsHasHelp(NULL,&flg1);
619:   if (flg1) {
620:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
621:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
622:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
623:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
624:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
625:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
626:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
627:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
628:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
629:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
630:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
631:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
632:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
633:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
634:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
635:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
636:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
637:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
638:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
639:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
640:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
641:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
642:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
643:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
644:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
645:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
646:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
647:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
648:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
649:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
650:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
651:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
652:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
653: #if defined(PETSC_USE_LOG)
654:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
655:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
656:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
657: #if defined(PETSC_HAVE_MPE)
658:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
659: #endif
660:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
661: #endif
662:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
663:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
664:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
665:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
666:   }

668: #if defined(PETSC_HAVE_POPEN)
669:   {
670:   char machine[128];
671:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
672:   if (flg1) {
673:     PetscPOpenSetMachine(machine);
674:   }
675:   }
676: #endif

678:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
679:   if (flg1) {
680:     PetscSleep(si);
681:   }

683:   PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
684:   if (flg1) {
685:     PetscStrstr(mname,"null",&f);
686:     if (f) {
687:       PetscInfoDeactivateClass(0);
688:     }
689:   }

691: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_CUDA)
692:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
693:   if (!flg3) {
694:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
695:   }
696: #endif
697: #if defined(PETSC_HAVE_VIENNACL)
698:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
699:   PetscViennaCLSynchronize = flg3;
700: #endif
701: #if defined(PETSC_HAVE_CUDA)
702:   PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
703:   PetscCUDASynchronize = flg3;
704: #endif

706: #if defined(PETSC_HAVE_VIENNACL)
707:   PetscViennaCLInit();
708: #endif

710:   return(0);
711: }