Actual source code: init.c
petsc-3.10.5 2019-03-28
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot known
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h>
10: #include <petsc/private/petscimpl.h>
11: #include <petscvalgrind.h>
12: #include <petscviewer.h>
14: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
15: #include <sys/sysinfo.h>
16: #endif
17: #if defined(PETSC_HAVE_UNISTD_H)
18: #include <unistd.h>
19: #endif
20: #if defined(PETSC_HAVE_CUDA)
21: #include <cuda_runtime.h>
22: #endif
24: #if defined(PETSC_HAVE_VIENNACL)
25: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
26: #endif
28: /* ------------------------Nasty global variables -------------------------------*/
29: /*
30: Indicates if PETSc started up MPI, or it was
31: already started before PETSc was initialized.
32: */
33: PetscBool PetscBeganMPI = PETSC_FALSE;
34: PetscBool PetscInitializeCalled = PETSC_FALSE;
35: PetscBool PetscFinalizeCalled = PETSC_FALSE;
36: PetscBool PetscCUDAInitialized = PETSC_FALSE;
38: PetscMPIInt PetscGlobalRank = -1;
39: PetscMPIInt PetscGlobalSize = -1;
41: #if defined(PETSC_HAVE_COMPLEX)
42: #if defined(PETSC_COMPLEX_INSTANTIATE)
43: template <> class std::complex<double>; /* instantiate complex template class */
44: #endif
45: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
46: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
47: MPI_Datatype MPIU_C_COMPLEX;
48: #endif
50: /*MC
51: PETSC_i - the imaginary number i
53: Synopsis:
54: #include <petscsys.h>
55: PetscComplex PETSC_i;
57: Level: beginner
59: Note:
60: Complex numbers are automatically available if PETSc located a working complex implementation
62: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
63: M*/
64: PetscComplex PETSC_i;
65: #endif
66: #if defined(PETSC_USE_REAL___FLOAT128)
67: MPI_Datatype MPIU___FLOAT128 = 0;
68: #if defined(PETSC_HAVE_COMPLEX)
69: MPI_Datatype MPIU___COMPLEX128 = 0;
70: #endif
71: #elif defined(PETSC_USE_REAL___FP16)
72: MPI_Datatype MPIU___FP16 = 0;
73: #endif
74: MPI_Datatype MPIU_2SCALAR = 0;
75: #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
76: MPI_Datatype MPIU_2INT = 0;
77: #endif
78: MPI_Datatype MPIU_BOOL;
79: MPI_Datatype MPIU_ENUM;
81: /*
82: Function that is called to display all error messages
83: */
84: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
85: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
86: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
87: /*
88: This is needed to turn on/off GPU synchronization
89: */
90: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
91: PetscBool PetscCUDASynchronize = PETSC_FALSE;
93: /* ------------------------------------------------------------------------------*/
94: /*
95: Optional file where all PETSc output from various prints is saved
96: */
97: PETSC_INTERN FILE *petsc_history;
98: FILE *petsc_history = NULL;
100: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
101: {
103: PetscMPIInt rank,size;
104: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
105: char version[256];
108: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
109: if (!rank) {
110: char arch[10];
111: int err;
113: PetscGetArchType(arch,10);
114: PetscGetDate(date,64);
115: PetscGetVersion(version,256);
116: MPI_Comm_size(PETSC_COMM_WORLD,&size);
117: if (filename) {
118: PetscFixFilename(filename,fname);
119: } else {
120: PetscGetHomeDirectory(pfile,240);
121: PetscStrcat(pfile,"/.petschistory");
122: PetscFixFilename(pfile,fname);
123: }
125: *fd = fopen(fname,"a");
126: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
128: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
129: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
130: PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
131: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
132: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
134: err = fflush(*fd);
135: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
136: }
137: return(0);
138: }
140: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
141: {
143: PetscMPIInt rank;
144: char date[64];
145: int err;
148: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
149: if (!rank) {
150: PetscGetDate(date,64);
151: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
152: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
153: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
154: err = fflush(*fd);
155: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
156: err = fclose(*fd);
157: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
158: }
159: return(0);
160: }
162: /* ------------------------------------------------------------------------------*/
164: /*
165: This is ugly and probably belongs somewhere else, but I want to
166: be able to put a true MPI abort error handler with command line args.
168: This is so MPI errors in the debugger will leave all the stack
169: frames. The default MP_Abort() cleans up and exits thus providing no useful information
170: in the debugger hence we call abort() instead of MPI_Abort().
171: */
173: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
174: {
176: (*PetscErrorPrintf)("MPI error %d\n",*flag);
177: abort();
178: }
180: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
181: {
185: (*PetscErrorPrintf)("MPI error %d\n",*flag);
186: PetscAttachDebugger();
187: if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
188: }
190: /*@C
191: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
192: wishes a clean exit somewhere deep in the program.
194: Collective on PETSC_COMM_WORLD
196: Options Database Keys are the same as for PetscFinalize()
198: Level: advanced
200: Note:
201: See PetscInitialize() for more general runtime options.
203: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
204: @*/
205: PetscErrorCode PetscEnd(void)
206: {
208: PetscFinalize();
209: exit(0);
210: return 0;
211: }
213: PetscBool PetscOptionsPublish = PETSC_FALSE;
214: PETSC_INTERN PetscErrorCode PetscSetUseTrMalloc_Private(void);
215: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
216: PETSC_INTERN PetscBool petscsetmallocvisited;
217: static char emacsmachinename[256];
219: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
220: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0;
222: /*@C
223: PetscSetHelpVersionFunctions - Sets functions that print help and version information
224: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
225: This routine enables a "higher-level" package that uses PETSc to print its messages first.
227: Input Parameter:
228: + help - the help function (may be NULL)
229: - version - the version function (may be NULL)
231: Level: developer
233: Concepts: package help message
235: @*/
236: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
237: {
239: PetscExternalHelpFunction = help;
240: PetscExternalVersionFunction = version;
241: return(0);
242: }
244: #if defined(PETSC_USE_LOG)
245: PETSC_INTERN PetscBool PetscObjectsLog;
246: #endif
248: PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void)
249: {
250: char string[64],mname[PETSC_MAX_PATH_LEN],*f;
251: MPI_Comm comm = PETSC_COMM_WORLD;
252: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
253: PetscErrorCode ierr;
254: PetscReal si;
255: PetscInt intensity;
256: int i;
257: PetscMPIInt rank;
258: char version[256],helpoptions[256];
259: #if !defined(PETSC_HAVE_THREADSAFETY)
260: PetscReal logthreshold;
261: #endif
262: #if defined(PETSC_USE_LOG)
263: PetscViewerFormat format;
264: PetscBool flg4 = PETSC_FALSE;
265: #endif
268: MPI_Comm_rank(comm,&rank);
270: #if !defined(PETSC_HAVE_THREADSAFETY)
271: /*
272: Setup the memory management; support for tracing malloc() usage
273: */
274: PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
275: logthreshold = 0.0;
276: PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
277: if (flg1) flg3 = PETSC_TRUE;
278: #if defined(PETSC_USE_DEBUG)
279: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
280: if ((!flg2 || flg1) && !petscsetmallocvisited) {
281: if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
282: /* turn off default -malloc if valgrind is being used */
283: PetscSetUseTrMalloc_Private();
284: }
285: }
286: #else
287: PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
288: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
289: if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
290: #endif
291: if (flg3) {
292: PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
293: }
294: PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
295: if (flg2) {PetscMallocSetCoalesce(flg1);}
296: flg1 = PETSC_FALSE;
297: PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
298: if (flg1) {
299: PetscSetUseTrMalloc_Private();
300: PetscMallocDebug(PETSC_TRUE);
301: }
302: flg1 = PETSC_FALSE;
303: PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
304: #if defined(PETSC_USE_DEBUG)
305: if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
306: PetscSetUseTrMalloc_Private();
307: PetscMallocSetDumpLog();
308: PetscMallocDebug(PETSC_TRUE);
309: }
310: #endif
311: flg1 = PETSC_FALSE;
312: PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
313: /* ignore this option if malloc is already set */
314: if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}
316: flg1 = PETSC_FALSE;
317: PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
318: if (!flg1) {
319: flg1 = PETSC_FALSE;
320: PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
321: }
322: if (flg1) {
323: PetscMemorySetGetMaximumUsage();
324: }
325: #endif
327: #if defined(PETSC_USE_LOG)
328: PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
329: #endif
331: /*
332: Set the display variable for graphics
333: */
334: PetscSetDisplay();
336: /*
337: Print the PETSc version information
338: */
339: PetscOptionsHasName(NULL,NULL,"-v",&flg1);
340: PetscOptionsHasName(NULL,NULL,"-version",&flg2);
341: PetscOptionsHasHelp(NULL,&flg3);
342: if (flg1 || flg2 || flg3) {
344: /*
345: Print "higher-level" package version message
346: */
347: if (PetscExternalVersionFunction) {
348: (*PetscExternalVersionFunction)(comm);
349: }
351: PetscGetVersion(version,256);
352: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
353: (*PetscHelpPrintf)(comm,"%s\n",version);
354: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
355: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
356: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
357: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
358: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
359: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
360: }
362: /*
363: Print "higher-level" package help message
364: */
365: if (flg3) {
366: if (PetscExternalHelpFunction) {
367: (*PetscExternalHelpFunction)(comm);
368: }
369: }
371: PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
372: if (flg1) {
373: PetscStrcmp(helpoptions,"intro",&flg2);
374: if (flg2) {
375: PetscOptionsDestroyDefault();
376: PetscFreeMPIResources();
377: MPI_Finalize();
378: exit(0);
379: }
380: }
382: /*
383: Setup the error handling
384: */
385: flg1 = PETSC_FALSE;
386: PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
387: if (flg1) {
388: MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
389: PetscPushErrorHandler(PetscAbortErrorHandler,0);
390: }
391: flg1 = PETSC_FALSE;
392: PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
393: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
394: flg1 = PETSC_FALSE;
395: PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
396: if (flg1) {
397: MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
398: }
399: flg1 = PETSC_FALSE;
400: PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
401: if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
402: flg1 = PETSC_FALSE;
403: PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
404: if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
405: PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
408: /*
409: Setup debugger information
410: */
411: PetscSetDefaultDebugger();
412: PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
413: if (flg1) {
414: MPI_Errhandler err_handler;
416: PetscSetDebuggerFromString(string);
417: MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
418: MPI_Comm_set_errhandler(comm,err_handler);
419: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
420: }
421: PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
422: if (flg1) { PetscSetDebugTerminal(string); }
423: PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
424: PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
425: if (flg1 || flg2) {
426: PetscMPIInt size;
427: PetscInt lsize,*nodes;
428: MPI_Errhandler err_handler;
429: /*
430: we have to make sure that all processors have opened
431: connections to all other processors, otherwise once the
432: debugger has stated it is likely to receive a SIGUSR1
433: and kill the program.
434: */
435: MPI_Comm_size(comm,&size);
436: if (size > 2) {
437: PetscMPIInt dummy = 0;
438: MPI_Status status;
439: for (i=0; i<size; i++) {
440: if (rank != i) {
441: MPI_Send(&dummy,1,MPI_INT,i,109,comm);
442: }
443: }
444: for (i=0; i<size; i++) {
445: if (rank != i) {
446: MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
447: }
448: }
449: }
450: /* check if this processor node should be in debugger */
451: PetscMalloc1(size,&nodes);
452: lsize = size;
453: PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
454: if (flag) {
455: for (i=0; i<lsize; i++) {
456: if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
457: }
458: }
459: if (!flag) {
460: PetscSetDebuggerFromString(string);
461: PetscPushErrorHandler(PetscAbortErrorHandler,0);
462: if (flg1) {
463: PetscAttachDebugger();
464: } else {
465: PetscStopForDebugger();
466: }
467: MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
468: MPI_Comm_set_errhandler(comm,err_handler);
469: }
470: PetscFree(nodes);
471: }
473: PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
474: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
476: /*
477: Setup profiling and logging
478: */
479: #if defined(PETSC_USE_INFO)
480: {
481: char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
482: PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
483: if (flg1 && logname[0]) {
484: PetscInfoAllow(PETSC_TRUE,logname);
485: } else if (flg1) {
486: PetscInfoAllow(PETSC_TRUE,NULL);
487: }
488: }
489: #endif
490: #if defined(PETSC_USE_LOG)
491: mname[0] = 0;
492: PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
493: if (flg1) {
494: if (mname[0]) {
495: PetscOpenHistoryFile(mname,&petsc_history);
496: } else {
497: PetscOpenHistoryFile(NULL,&petsc_history);
498: }
499: }
501: PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);
503: #if defined(PETSC_HAVE_MPE)
504: flg1 = PETSC_FALSE;
505: PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
506: if (flg1) {PetscLogMPEBegin();}
507: #endif
508: flg1 = PETSC_FALSE;
509: flg3 = PETSC_FALSE;
510: PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
511: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
512: if (flg1) { PetscLogAllBegin(); }
513: else if (flg3) { PetscLogDefaultBegin();}
515: PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
516: if (flg1) {
517: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
518: FILE *file;
519: if (mname[0]) {
520: PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
521: PetscFixFilename(name,fname);
522: file = fopen(fname,"w");
523: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
524: } else file = PETSC_STDOUT;
525: PetscLogTraceBegin(file);
526: }
528: PetscOptionsGetViewer(comm,NULL,"-log_view",NULL,&format,&flg4);
529: if (flg4) {
530: if (format == PETSC_VIEWER_ASCII_XML) {
531: PetscLogNestedBegin();
532: } else {
533: PetscLogDefaultBegin();
534: }
535: }
536: if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
537: PetscReal threshold = PetscRealConstant(0.01);
538: PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
539: if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
540: }
541: #endif
543: PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
545: #if defined(PETSC_HAVE_CUDA)
546: PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
547: if (flg1) {
548: struct cudaDeviceProp prop;
549: int devCount;
550: PetscInt device;
551: cudaError_t err = cudaSuccess;
553: err = cudaGetDeviceCount(&devCount);
554: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
555: for (device = 0; device < devCount; ++device) {
556: err = cudaGetDeviceProperties(&prop, (int)device);
557: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
558: PetscPrintf(comm, "CUDA device %D: %s\n", device, prop.name);
559: }
560: }
561: if (!PetscCUDAInitialized) {
562: PetscMPIInt size;
563: MPI_Comm_size(comm,&size);
564: if (size>1) {
565: int devCount;
566: PetscInt device;
567: PetscMPIInt rank;
568: cudaError_t err = cudaSuccess;
570: /* check to see if we force multiple ranks to hit the same GPU */
571: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
572: if (flg1) {
573: err = cudaSetDevice((int)device);
574: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
575: } else {
576: /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */
578: /* First get the device count */
579: err = cudaGetDeviceCount(&devCount);
580: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
582: /* next determine the rank and then set the device via a mod */
583: MPI_Comm_rank(comm,&rank);
584: device = rank % devCount;
585: err = cudaSetDevice((int)device);
586: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
587: }
589: /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
590: multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
591: are ok. */
592: err = cudaSetDeviceFlags(cudaDeviceMapHost);
593: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
594: } else {
595: PetscInt device;
596: cudaError_t err = cudaSuccess;
598: /* the code below works for serial GPU simulations */
599: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
600: if (flg1) {
601: err = cudaSetDevice((int)device);
602: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
603: }
605: /* set the device flags so that it can map host memory ... here, we error check. */
606: err = cudaSetDeviceFlags(cudaDeviceMapHost);
607: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
608: }
610: PetscCUDAInitialized = PETSC_TRUE;
611: }
612: #endif
615: /*
616: Print basic help message
617: */
618: PetscOptionsHasHelp(NULL,&flg1);
619: if (flg1) {
620: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
621: (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
622: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
623: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
624: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
625: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
626: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
627: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
628: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
629: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
630: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
631: (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
632: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
633: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
634: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
635: (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
636: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
637: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
638: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
639: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
640: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
641: (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
642: (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
643: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
644: (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
645: (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
646: (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
647: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
648: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
649: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
650: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
651: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
652: (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
653: #if defined(PETSC_USE_LOG)
654: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
655: (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
656: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
657: #if defined(PETSC_HAVE_MPE)
658: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
659: #endif
660: (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
661: #endif
662: (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
663: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
664: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
665: (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
666: }
668: #if defined(PETSC_HAVE_POPEN)
669: {
670: char machine[128];
671: PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
672: if (flg1) {
673: PetscPOpenSetMachine(machine);
674: }
675: }
676: #endif
678: PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
679: if (flg1) {
680: PetscSleep(si);
681: }
683: PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
684: if (flg1) {
685: PetscStrstr(mname,"null",&f);
686: if (f) {
687: PetscInfoDeactivateClass(0);
688: }
689: }
691: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA)
692: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
693: if (!flg3) {
694: PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
695: }
696: #endif
697: #if defined(PETSC_HAVE_VIENNACL)
698: PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
699: PetscViennaCLSynchronize = flg3;
700: #endif
701: #if defined(PETSC_HAVE_VECCUDA)
702: PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
703: PetscCUDASynchronize = flg3;
704: #endif
706: #if defined(PETSC_HAVE_VIENNACL)
707: PetscViennaCLInit();
708: #endif
710: return(0);
711: }