Actual source code: init.c

petsc-3.7.3 2016-08-01
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>        /*I  "petscsys.h"   I*/
 10: #include <petscvalgrind.h>
 11: #include <petscviewer.h>

 13: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 14: #include <sys/sysinfo.h>
 15: #endif
 16: #if defined(PETSC_HAVE_UNISTD_H)
 17: #include <unistd.h>
 18: #endif
 19: #if defined(PETSC_HAVE_CUDA)
 20: #include <cuda_runtime.h>
 21: #endif

 23: /* ------------------------Nasty global variables -------------------------------*/
 24: /*
 25:      Indicates if PETSc started up MPI, or it was
 26:    already started before PETSc was initialized.
 27: */
 28: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 29: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 30: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 31: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 33: PetscMPIInt PetscGlobalRank       = -1;
 34: PetscMPIInt PetscGlobalSize       = -1;

 36: #if defined(PETSC_HAVE_COMPLEX)
 37: #if defined(PETSC_COMPLEX_INSTANTIATE)
 38: template <> class std::complex<double>; /* instantiate complex template class */
 39: #endif
 40: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 41: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 42: MPI_Datatype MPIU_C_COMPLEX;
 43: #endif

 45: /*MC
 46:    PETSC_i - the imaginary number i

 48:    Synopsis:
 49:    #include <petscsys.h>
 50:    PetscComplex PETSC_i;

 52:    Level: beginner

 54:    Note:
 55:    Complex numbers are automatically available if PETSc located a working complex implementation

 57: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 58: M*/
 59: PetscComplex PETSC_i;
 60: #endif
 61: #if defined(PETSC_USE_REAL___FLOAT128)
 62: MPI_Datatype MPIU___FLOAT128 = 0;
 63: #if defined(PETSC_HAVE_COMPLEX)
 64: MPI_Datatype MPIU___COMPLEX128 = 0;
 65: #endif
 66: #endif
 67: MPI_Datatype MPIU_2SCALAR = 0;
 68: #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
 69: MPI_Datatype MPIU_2INT = 0;
 70: #endif
 71: MPI_Datatype MPIU_BOOL;
 72: MPI_Datatype MPIU_ENUM;

 74: /*
 75:        Function that is called to display all error messages
 76: */
 77: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 78: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 79: #if defined(PETSC_HAVE_MATLAB_ENGINE)
 80: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintf_Matlab;
 81: #else
 82: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 83: #endif
 84: /*
 85:   This is needed to turn on/off GPU synchronization
 86: */
 87: PetscBool PetscCUSPSynchronize = PETSC_FALSE;
 88: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
 89: PetscBool PetscCUDASynchronize = PETSC_FALSE;

 91: /* ------------------------------------------------------------------------------*/
 92: /*
 93:    Optional file where all PETSc output from various prints is saved
 94: */
 95: FILE *petsc_history = NULL;

 99: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
100: {
102:   PetscMPIInt    rank,size;
103:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
104:   char           version[256];

107:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
108:   if (!rank) {
109:     char        arch[10];
110:     int         err;

112:     PetscGetArchType(arch,10);
113:     PetscGetDate(date,64);
114:     PetscGetVersion(version,256);
115:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
116:     if (filename) {
117:       PetscFixFilename(filename,fname);
118:     } else {
119:       PetscGetHomeDirectory(pfile,240);
120:       PetscStrcat(pfile,"/.petschistory");
121:       PetscFixFilename(pfile,fname);
122:     }

124:     *fd = fopen(fname,"a");
125:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

127:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
128:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
129:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
130:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
131:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");

133:     err = fflush(*fd);
134:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
135:   }
136:   return(0);
137: }

141: PetscErrorCode  PetscCloseHistoryFile(FILE **fd)
142: {
144:   PetscMPIInt    rank;
145:   char           date[64];
146:   int            err;

149:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
150:   if (!rank) {
151:     PetscGetDate(date,64);
152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
153:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
154:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
155:     err  = fflush(*fd);
156:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
157:     err = fclose(*fd);
158:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
159:   }
160:   return(0);
161: }

163: /* ------------------------------------------------------------------------------*/

165: /*
166:    This is ugly and probably belongs somewhere else, but I want to
167:   be able to put a true MPI abort error handler with command line args.

169:     This is so MPI errors in the debugger will leave all the stack
170:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
171:   in the debugger hence we call abort() instead of MPI_Abort().
172: */

176: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
177: {
179:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
180:   abort();
181: }

185: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
186: {

190:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
191:   PetscAttachDebugger();
192:   if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
193: }

197: /*@C
198:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
199:      wishes a clean exit somewhere deep in the program.

201:    Collective on PETSC_COMM_WORLD

203:    Options Database Keys are the same as for PetscFinalize()

205:    Level: advanced

207:    Note:
208:    See PetscInitialize() for more general runtime options.

210: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
211: @*/
212: PetscErrorCode  PetscEnd(void)
213: {
215:   PetscFinalize();
216:   exit(0);
217:   return 0;
218: }

220: PetscBool PetscOptionsPublish = PETSC_FALSE;
221: extern PetscErrorCode PetscSetUseTrMalloc_Private(void);
222: extern PetscBool      petscsetmallocvisited;
223: static char           emacsmachinename[256];

225: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
226: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

230: /*@C
231:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
232:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
233:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

235:    Input Parameter:
236: +  help - the help function (may be NULL)
237: -  version - the version function (may be NULL)

239:    Level: developer

241:    Concepts: package help message

243: @*/
244: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
245: {
247:   PetscExternalHelpFunction    = help;
248:   PetscExternalVersionFunction = version;
249:   return(0);
250: }

252: #if defined(PETSC_USE_LOG)
253: extern PetscBool   PetscObjectsLog;
254: #endif

258: PetscErrorCode  PetscOptionsCheckInitial_Private(void)
259: {
260:   char              string[64],mname[PETSC_MAX_PATH_LEN],*f;
261:   MPI_Comm          comm = PETSC_COMM_WORLD;
262:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
263:   PetscErrorCode    ierr;
264:   PetscReal         si;
265:   PetscInt          intensity;
266:   int               i;
267:   PetscMPIInt       rank;
268:   char              version[256];
269: #if !defined(PETSC_HAVE_THREADSAFETY)
270:   PetscReal         logthreshold;
271: #endif
272: #if defined(PETSC_USE_LOG)
273:   PetscViewerFormat format;
274:   PetscBool         flg4 = PETSC_FALSE;
275: #endif
276: 
278:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);

280: #if !defined(PETSC_HAVE_THREADSAFETY)
281:   /*
282:       Setup the memory management; support for tracing malloc() usage
283:   */
284:   PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
285:   logthreshold = 0.0;
286:   PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
287:   if (flg1) flg3 = PETSC_TRUE;
288: #if defined(PETSC_USE_DEBUG)
289:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
290:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
291:     if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
292:       /* turn off default -malloc if valgrind is being used */
293:       PetscSetUseTrMalloc_Private();
294:     }
295:   }
296: #else
297:   PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
298:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
299:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
300: #endif
301:   if (flg3) {
302:     PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
303:   }
304:   flg1 = PETSC_FALSE;
305:   PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
306:   if (flg1) {
307:     PetscSetUseTrMalloc_Private();
308:     PetscMallocDebug(PETSC_TRUE);
309:   }
310:   flg1 = PETSC_FALSE;
311:   PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
312: #if defined(PETSC_USE_DEBUG)
313:   if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
314:     PetscSetUseTrMalloc_Private();
315:     PetscMallocSetDumpLog();
316:     PetscMallocDebug(PETSC_TRUE);
317:   }
318: #endif

320:   flg1 = PETSC_FALSE;
321:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
322:   if (!flg1) {
323:     flg1 = PETSC_FALSE;
324:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
325:   }
326:   if (flg1) {
327:     PetscMemorySetGetMaximumUsage();
328:   }
329: #endif

331: #if defined(PETSC_USE_LOG)
332:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
333: #endif

335:   /*
336:       Set the display variable for graphics
337:   */
338:   PetscSetDisplay();

340:   /*
341:       Print the PETSc version information
342:   */
343:   PetscOptionsHasName(NULL,NULL,"-v",&flg1);
344:   PetscOptionsHasName(NULL,NULL,"-version",&flg2);
345:   PetscOptionsHasName(NULL,NULL,"-help",&flg3);
346:   if (flg1 || flg2 || flg3) {

348:     /*
349:        Print "higher-level" package version message
350:     */
351:     if (PetscExternalVersionFunction) {
352:       (*PetscExternalVersionFunction)(comm);
353:     }

355:     PetscGetVersion(version,256);
356:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
357: ------------------------------\n");
358:     (*PetscHelpPrintf)(comm,"%s\n",version);
359:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
360:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
361:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
362:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
363:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
364:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
365: ------------------------------\n");
366:   }

368:   /*
369:        Print "higher-level" package help message
370:   */
371:   if (flg3) {
372:     if (PetscExternalHelpFunction) {
373:       (*PetscExternalHelpFunction)(comm);
374:     }
375:   }

377:   /*
378:       Setup the error handling
379:   */
380:   flg1 = PETSC_FALSE;
381:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
382:   if (flg1) {
383:     MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_ARE_FATAL);
384:     PetscPushErrorHandler(PetscAbortErrorHandler,0);
385:   }
386:   flg1 = PETSC_FALSE;
387:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
388:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
389:   flg1 = PETSC_FALSE;
390:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
391:   if (flg1) {
392:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
393:   }
394:   flg1 = PETSC_FALSE;
395:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
396:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
397:   flg1 = PETSC_FALSE;
398:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
399:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
400:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);

403:   /*
404:       Setup debugger information
405:   */
406:   PetscSetDefaultDebugger();
407:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
408:   if (flg1) {
409:     MPI_Errhandler err_handler;

411:     PetscSetDebuggerFromString(string);
412:     MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
413:     MPI_Comm_set_errhandler(comm,err_handler);
414:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
415:   }
416:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
417:   if (flg1) { PetscSetDebugTerminal(string); }
418:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
419:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
420:   if (flg1 || flg2) {
421:     PetscMPIInt    size;
422:     PetscInt       lsize,*nodes;
423:     MPI_Errhandler err_handler;
424:     /*
425:        we have to make sure that all processors have opened
426:        connections to all other processors, otherwise once the
427:        debugger has stated it is likely to receive a SIGUSR1
428:        and kill the program.
429:     */
430:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
431:     if (size > 2) {
432:       PetscMPIInt dummy = 0;
433:       MPI_Status  status;
434:       for (i=0; i<size; i++) {
435:         if (rank != i) {
436:           MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
437:         }
438:       }
439:       for (i=0; i<size; i++) {
440:         if (rank != i) {
441:           MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
442:         }
443:       }
444:     }
445:     /* check if this processor node should be in debugger */
446:     PetscMalloc1(size,&nodes);
447:     lsize = size;
448:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
449:     if (flag) {
450:       for (i=0; i<lsize; i++) {
451:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
452:       }
453:     }
454:     if (!flag) {
455:       PetscSetDebuggerFromString(string);
456:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
457:       if (flg1) {
458:         PetscAttachDebugger();
459:       } else {
460:         PetscStopForDebugger();
461:       }
462:       MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
463:       MPI_Comm_set_errhandler(comm,err_handler);
464:     }
465:     PetscFree(nodes);
466:   }

468:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
469:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

471:   /*
472:         Setup profiling and logging
473:   */
474: #if defined(PETSC_USE_INFO)
475:   {
476:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
477:     PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
478:     if (flg1 && logname[0]) {
479:       PetscInfoAllow(PETSC_TRUE,logname);
480:     } else if (flg1) {
481:       PetscInfoAllow(PETSC_TRUE,NULL);
482:     }
483:   }
484: #endif
485: #if defined(PETSC_USE_LOG)
486:   mname[0] = 0;
487:   PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
488:   if (flg1) {
489:     if (mname[0]) {
490:       PetscOpenHistoryFile(mname,&petsc_history);
491:     } else {
492:       PetscOpenHistoryFile(NULL,&petsc_history);
493:     }
494:   }
495: #if defined(PETSC_HAVE_MPE)
496:   flg1 = PETSC_FALSE;
497:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
498:   if (flg1) {PetscLogMPEBegin();}
499: #endif
500:   flg1 = PETSC_FALSE;
501:   flg3 = PETSC_FALSE;
502:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
503:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
504:   if (flg1)                      { PetscLogAllBegin(); }
505:   else if (flg3)                 { PetscLogDefaultBegin();}

507:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
508:   if (flg1) {
509:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
510:     FILE *file;
511:     if (mname[0]) {
512:       sprintf(name,"%s.%d",mname,rank);
513:       PetscFixFilename(name,fname);
514:       file = fopen(fname,"w");
515:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
516:     } else file = PETSC_STDOUT;
517:     PetscLogTraceBegin(file);
518:   }

520:   PetscOptionsGetViewer(PETSC_COMM_WORLD,NULL,"-log_view",NULL,&format,&flg4);
521:   if (flg4) {
522:     if (format == PETSC_VIEWER_ASCII_XML){
523:       PetscLogNestedBegin();
524:     } else {
525:       PetscLogDefaultBegin();
526:     }
527:   }
528: #endif

530:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

532: #if defined(PETSC_HAVE_CUDA)
533:   PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
534:   if (flg1) {
535:     struct cudaDeviceProp prop;
536:     int                   devCount;
537:     PetscInt              device;
538:     cudaError_t           err = cudaSuccess;

540:     err = cudaGetDeviceCount(&devCount);
541:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
542:     for (device = 0; device < devCount; ++device) {
543:       err = cudaGetDeviceProperties(&prop, (int)device);
544:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
545:       PetscPrintf(PETSC_COMM_WORLD, "CUDA device %D: %s\n", device, prop.name);
546:     }
547:   }
548:   if (!PetscCUDAInitialized) {
549:     PetscMPIInt size;
550:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
551:     if (size>1) {
552:       int         devCount;
553:       PetscInt    device;
554:       PetscMPIInt rank;
555:       cudaError_t err = cudaSuccess;

557:       /* check to see if we force multiple ranks to hit the same GPU */
558:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
559:       if (flg1) {
560:         err = cudaSetDevice((int)device);
561:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
562:       } else {
563:         /* we're not using the same GPU on multiple MPI threads. So try to allocated different   GPUs to different processes */

565:         /* First get the device count */
566:         err   = cudaGetDeviceCount(&devCount);
567:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));

569:         /* next determine the rank and then set the device via a mod */
570:         MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
571:         device = rank % devCount;
572:         err    = cudaSetDevice((int)device);
573:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
574:       }

576:       /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
577:        multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
578:        are ok. */
579:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
580:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
581:     } else {
582:       PetscInt    device;
583:       cudaError_t err = cudaSuccess;

585:       /* the code below works for serial GPU simulations */
586:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
587:       if (flg1) {
588:         err = cudaSetDevice((int)device);
589:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
590:       }

592:       /* set the device flags so that it can map host memory ... here, we error check. */
593:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
594:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
595:     }

597:     PetscCUDAInitialized = PETSC_TRUE;
598:   }
599: #endif


602:   /*
603:        Print basic help message
604:   */
605:   PetscOptionsHasName(NULL,NULL,"-help",&flg1);
606:   if (flg1) {
607:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
608:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
609:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
610:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
611:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
612:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
613:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
614:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
615:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
616:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
617:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
618:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
619:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
620:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
621:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
622:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
623:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
624:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
625:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
626:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
627:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
628:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
629:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
630:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
631:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
632:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
633:     (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
634:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
635:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
636:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
637:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
638:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
639:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
640: #if defined(PETSC_USE_LOG)
641:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
642:     (*PetscHelpPrintf)(comm," -log[_summary _summary_python]: logging objects and events\n");
643:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
644: #if defined(PETSC_HAVE_MPE)
645:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
646: #endif
647:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
648: #endif
649:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
650:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
651:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
652:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
653:   }

655: #if defined(PETSC_HAVE_POPEN)
656:   {
657:   char machine[128];
658:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
659:   if (flg1) {
660:     PetscPOpenSetMachine(machine);
661:   }
662:   }
663: #endif

665:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
666:   if (flg1) {
667:     PetscSleep(si);
668:   }

670:   PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
671:   if (flg1) {
672:     PetscStrstr(mname,"null",&f);
673:     if (f) {
674:       PetscInfoDeactivateClass(0);
675:     }
676:   }

678: #if defined(PETSC_HAVE_CUSP) || defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA)
679:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
680:   if (!flg3) {
681:   PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
682:   }
683: #endif
684: #if defined(PETSC_HAVE_CUSP)
685:   PetscOptionsGetBool(NULL,NULL,"-cusp_synchronize",&flg3,NULL);
686:   PetscCUSPSynchronize = flg3;
687: #elif defined(PETSC_HAVE_VIENNACL)
688:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
689:   PetscViennaCLSynchronize = flg3;
690: #elif defined(PETSC_HAVE_VECCUDA)
691:   PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
692:   PetscCUDASynchronize = flg3;
693: #endif

695:   return(0);
696: }