Actual source code: init.c
petsc-3.9.4 2018-09-11
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot known
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h>
10: #include <petsc/private/petscimpl.h>
11: #include <petscvalgrind.h>
12: #include <petscviewer.h>
14: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
15: #include <sys/sysinfo.h>
16: #endif
17: #if defined(PETSC_HAVE_UNISTD_H)
18: #include <unistd.h>
19: #endif
20: #if defined(PETSC_HAVE_CUDA)
21: #include <cuda_runtime.h>
22: #endif
24: #if defined(PETSC_HAVE_VIENNACL)
25: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
26: #endif
28: /* ------------------------Nasty global variables -------------------------------*/
29: /*
30: Indicates if PETSc started up MPI, or it was
31: already started before PETSc was initialized.
32: */
33: PetscBool PetscBeganMPI = PETSC_FALSE;
34: PetscBool PetscInitializeCalled = PETSC_FALSE;
35: PetscBool PetscFinalizeCalled = PETSC_FALSE;
36: PetscBool PetscCUDAInitialized = PETSC_FALSE;
38: PetscMPIInt PetscGlobalRank = -1;
39: PetscMPIInt PetscGlobalSize = -1;
41: #if defined(PETSC_HAVE_COMPLEX)
42: #if defined(PETSC_COMPLEX_INSTANTIATE)
43: template <> class std::complex<double>; /* instantiate complex template class */
44: #endif
45: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
46: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
47: MPI_Datatype MPIU_C_COMPLEX;
48: #endif
50: /*MC
51: PETSC_i - the imaginary number i
53: Synopsis:
54: #include <petscsys.h>
55: PetscComplex PETSC_i;
57: Level: beginner
59: Note:
60: Complex numbers are automatically available if PETSc located a working complex implementation
62: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
63: M*/
64: PetscComplex PETSC_i;
65: #endif
66: #if defined(PETSC_USE_REAL___FLOAT128)
67: MPI_Datatype MPIU___FLOAT128 = 0;
68: #if defined(PETSC_HAVE_COMPLEX)
69: MPI_Datatype MPIU___COMPLEX128 = 0;
70: #endif
71: #elif defined(PETSC_USE_REAL___FP16)
72: MPI_Datatype MPIU___FP16 = 0;
73: #endif
74: MPI_Datatype MPIU_2SCALAR = 0;
75: #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
76: MPI_Datatype MPIU_2INT = 0;
77: #endif
78: MPI_Datatype MPIU_BOOL;
79: MPI_Datatype MPIU_ENUM;
81: /*
82: Function that is called to display all error messages
83: */
84: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
85: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
86: #if defined(PETSC_HAVE_MATLAB_ENGINE)
87: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintf_Matlab;
88: #else
89: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
90: #endif
91: /*
92: This is needed to turn on/off GPU synchronization
93: */
94: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
95: PetscBool PetscCUDASynchronize = PETSC_FALSE;
97: /* ------------------------------------------------------------------------------*/
98: /*
99: Optional file where all PETSc output from various prints is saved
100: */
101: FILE *petsc_history = NULL;
103: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
104: {
106: PetscMPIInt rank,size;
107: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
108: char version[256];
111: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
112: if (!rank) {
113: char arch[10];
114: int err;
116: PetscGetArchType(arch,10);
117: PetscGetDate(date,64);
118: PetscGetVersion(version,256);
119: MPI_Comm_size(PETSC_COMM_WORLD,&size);
120: if (filename) {
121: PetscFixFilename(filename,fname);
122: } else {
123: PetscGetHomeDirectory(pfile,240);
124: PetscStrcat(pfile,"/.petschistory");
125: PetscFixFilename(pfile,fname);
126: }
128: *fd = fopen(fname,"a");
129: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
131: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
132: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
133: PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
134: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
135: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
137: err = fflush(*fd);
138: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
139: }
140: return(0);
141: }
143: PetscErrorCode PetscCloseHistoryFile(FILE **fd)
144: {
146: PetscMPIInt rank;
147: char date[64];
148: int err;
151: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
152: if (!rank) {
153: PetscGetDate(date,64);
154: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
155: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
156: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
157: err = fflush(*fd);
158: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
159: err = fclose(*fd);
160: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
161: }
162: return(0);
163: }
165: /* ------------------------------------------------------------------------------*/
167: /*
168: This is ugly and probably belongs somewhere else, but I want to
169: be able to put a true MPI abort error handler with command line args.
171: This is so MPI errors in the debugger will leave all the stack
172: frames. The default MP_Abort() cleans up and exits thus providing no useful information
173: in the debugger hence we call abort() instead of MPI_Abort().
174: */
176: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
177: {
179: (*PetscErrorPrintf)("MPI error %d\n",*flag);
180: abort();
181: }
183: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
184: {
188: (*PetscErrorPrintf)("MPI error %d\n",*flag);
189: PetscAttachDebugger();
190: if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
191: }
193: /*@C
194: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
195: wishes a clean exit somewhere deep in the program.
197: Collective on PETSC_COMM_WORLD
199: Options Database Keys are the same as for PetscFinalize()
201: Level: advanced
203: Note:
204: See PetscInitialize() for more general runtime options.
206: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
207: @*/
208: PetscErrorCode PetscEnd(void)
209: {
211: PetscFinalize();
212: exit(0);
213: return 0;
214: }
216: PetscBool PetscOptionsPublish = PETSC_FALSE;
217: extern PetscErrorCode PetscSetUseTrMalloc_Private(void);
218: extern PetscErrorCode PetscSetUseHBWMalloc_Private(void);
219: extern PetscBool petscsetmallocvisited;
220: static char emacsmachinename[256];
222: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
223: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0;
225: /*@C
226: PetscSetHelpVersionFunctions - Sets functions that print help and version information
227: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
228: This routine enables a "higher-level" package that uses PETSc to print its messages first.
230: Input Parameter:
231: + help - the help function (may be NULL)
232: - version - the version function (may be NULL)
234: Level: developer
236: Concepts: package help message
238: @*/
239: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
240: {
242: PetscExternalHelpFunction = help;
243: PetscExternalVersionFunction = version;
244: return(0);
245: }
247: #if defined(PETSC_USE_LOG)
248: extern PetscBool PetscObjectsLog;
249: #endif
251: PetscErrorCode PetscOptionsCheckInitial_Private(void)
252: {
253: char string[64],mname[PETSC_MAX_PATH_LEN],*f;
254: MPI_Comm comm = PETSC_COMM_WORLD;
255: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
256: PetscErrorCode ierr;
257: PetscReal si;
258: PetscInt intensity;
259: int i;
260: PetscMPIInt rank;
261: char version[256],helpoptions[256];
262: #if !defined(PETSC_HAVE_THREADSAFETY)
263: PetscReal logthreshold;
264: #endif
265: #if defined(PETSC_USE_LOG)
266: PetscViewerFormat format;
267: PetscBool flg4 = PETSC_FALSE;
268: #endif
271: MPI_Comm_rank(comm,&rank);
273: #if !defined(PETSC_HAVE_THREADSAFETY)
274: /*
275: Setup the memory management; support for tracing malloc() usage
276: */
277: PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
278: logthreshold = 0.0;
279: PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
280: if (flg1) flg3 = PETSC_TRUE;
281: #if defined(PETSC_USE_DEBUG)
282: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
283: if ((!flg2 || flg1) && !petscsetmallocvisited) {
284: if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
285: /* turn off default -malloc if valgrind is being used */
286: PetscSetUseTrMalloc_Private();
287: }
288: }
289: #else
290: PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
291: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
292: if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
293: #endif
294: if (flg3) {
295: PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
296: }
297: PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
298: if (flg2) {PetscMallocSetCoalesce(flg1);}
299: flg1 = PETSC_FALSE;
300: PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
301: if (flg1) {
302: PetscSetUseTrMalloc_Private();
303: PetscMallocDebug(PETSC_TRUE);
304: }
305: flg1 = PETSC_FALSE;
306: PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
307: #if defined(PETSC_USE_DEBUG)
308: if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
309: PetscSetUseTrMalloc_Private();
310: PetscMallocSetDumpLog();
311: PetscMallocDebug(PETSC_TRUE);
312: }
313: #endif
314: flg1 = PETSC_FALSE;
315: PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
316: /* ignore this option if malloc is already set */
317: if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}
319: flg1 = PETSC_FALSE;
320: PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
321: if (!flg1) {
322: flg1 = PETSC_FALSE;
323: PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
324: }
325: if (flg1) {
326: PetscMemorySetGetMaximumUsage();
327: }
328: #endif
330: #if defined(PETSC_USE_LOG)
331: PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
332: #endif
334: /*
335: Set the display variable for graphics
336: */
337: PetscSetDisplay();
339: /*
340: Print the PETSc version information
341: */
342: PetscOptionsHasName(NULL,NULL,"-v",&flg1);
343: PetscOptionsHasName(NULL,NULL,"-version",&flg2);
344: PetscOptionsHasName(NULL,NULL,"-help",&flg3);
345: if (flg1 || flg2 || flg3) {
347: /*
348: Print "higher-level" package version message
349: */
350: if (PetscExternalVersionFunction) {
351: (*PetscExternalVersionFunction)(comm);
352: }
354: PetscGetVersion(version,256);
355: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
356: (*PetscHelpPrintf)(comm,"%s\n",version);
357: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
358: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
359: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
360: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
361: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
362: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
363: }
365: /*
366: Print "higher-level" package help message
367: */
368: if (flg3) {
369: if (PetscExternalHelpFunction) {
370: (*PetscExternalHelpFunction)(comm);
371: }
372: }
374: PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
375: if (flg1) {
376: PetscStrcmp(helpoptions,"intro",&flg2);
377: if (flg2) {
378: PetscOptionsDestroyDefault();
379: PetscFreeMPIResources();
380: MPI_Finalize();
381: exit(0);
382: }
383: }
385: /*
386: Setup the error handling
387: */
388: flg1 = PETSC_FALSE;
389: PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
390: if (flg1) {
391: MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
392: PetscPushErrorHandler(PetscAbortErrorHandler,0);
393: }
394: flg1 = PETSC_FALSE;
395: PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
396: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
397: flg1 = PETSC_FALSE;
398: PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
399: if (flg1) {
400: MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
401: }
402: flg1 = PETSC_FALSE;
403: PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
404: if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
405: flg1 = PETSC_FALSE;
406: PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
407: if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
408: PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
411: /*
412: Setup debugger information
413: */
414: PetscSetDefaultDebugger();
415: PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
416: if (flg1) {
417: MPI_Errhandler err_handler;
419: PetscSetDebuggerFromString(string);
420: MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
421: MPI_Comm_set_errhandler(comm,err_handler);
422: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
423: }
424: PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
425: if (flg1) { PetscSetDebugTerminal(string); }
426: PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
427: PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
428: if (flg1 || flg2) {
429: PetscMPIInt size;
430: PetscInt lsize,*nodes;
431: MPI_Errhandler err_handler;
432: /*
433: we have to make sure that all processors have opened
434: connections to all other processors, otherwise once the
435: debugger has stated it is likely to receive a SIGUSR1
436: and kill the program.
437: */
438: MPI_Comm_size(comm,&size);
439: if (size > 2) {
440: PetscMPIInt dummy = 0;
441: MPI_Status status;
442: for (i=0; i<size; i++) {
443: if (rank != i) {
444: MPI_Send(&dummy,1,MPI_INT,i,109,comm);
445: }
446: }
447: for (i=0; i<size; i++) {
448: if (rank != i) {
449: MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
450: }
451: }
452: }
453: /* check if this processor node should be in debugger */
454: PetscMalloc1(size,&nodes);
455: lsize = size;
456: PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
457: if (flag) {
458: for (i=0; i<lsize; i++) {
459: if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
460: }
461: }
462: if (!flag) {
463: PetscSetDebuggerFromString(string);
464: PetscPushErrorHandler(PetscAbortErrorHandler,0);
465: if (flg1) {
466: PetscAttachDebugger();
467: } else {
468: PetscStopForDebugger();
469: }
470: MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
471: MPI_Comm_set_errhandler(comm,err_handler);
472: }
473: PetscFree(nodes);
474: }
476: PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
477: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
479: /*
480: Setup profiling and logging
481: */
482: #if defined(PETSC_USE_INFO)
483: {
484: char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
485: PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
486: if (flg1 && logname[0]) {
487: PetscInfoAllow(PETSC_TRUE,logname);
488: } else if (flg1) {
489: PetscInfoAllow(PETSC_TRUE,NULL);
490: }
491: }
492: #endif
493: #if defined(PETSC_USE_LOG)
494: mname[0] = 0;
495: PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
496: if (flg1) {
497: if (mname[0]) {
498: PetscOpenHistoryFile(mname,&petsc_history);
499: } else {
500: PetscOpenHistoryFile(NULL,&petsc_history);
501: }
502: }
503: #if defined(PETSC_HAVE_MPE)
504: flg1 = PETSC_FALSE;
505: PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
506: if (flg1) {PetscLogMPEBegin();}
507: #endif
508: flg1 = PETSC_FALSE;
509: flg3 = PETSC_FALSE;
510: PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
511: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
512: if (flg1) { PetscLogAllBegin(); }
513: else if (flg3) { PetscLogDefaultBegin();}
515: PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
516: if (flg1) {
517: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
518: FILE *file;
519: if (mname[0]) {
520: PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
521: PetscFixFilename(name,fname);
522: file = fopen(fname,"w");
523: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
524: } else file = PETSC_STDOUT;
525: PetscLogTraceBegin(file);
526: }
528: PetscOptionsGetViewer(comm,NULL,"-log_view",NULL,&format,&flg4);
529: if (flg4) {
530: if (format == PETSC_VIEWER_ASCII_XML) {
531: PetscLogNestedBegin();
532: } else {
533: PetscLogDefaultBegin();
534: }
535: }
536: if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
537: PetscReal threshold = PetscRealConstant(0.01);
538: PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
539: if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
540: }
541: #endif
543: PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
546: #if defined(PETSC_HAVE_CUDA)
547: PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
548: if (flg1) {
549: struct cudaDeviceProp prop;
550: int devCount;
551: PetscInt device;
552: cudaError_t err = cudaSuccess;
554: err = cudaGetDeviceCount(&devCount);
555: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
556: for (device = 0; device < devCount; ++device) {
557: err = cudaGetDeviceProperties(&prop, (int)device);
558: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
559: PetscPrintf(comm, "CUDA device %D: %s\n", device, prop.name);
560: }
561: }
562: if (!PetscCUDAInitialized) {
563: PetscMPIInt size;
564: MPI_Comm_size(comm,&size);
565: if (size>1) {
566: int devCount;
567: PetscInt device;
568: PetscMPIInt rank;
569: cudaError_t err = cudaSuccess;
571: /* check to see if we force multiple ranks to hit the same GPU */
572: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
573: if (flg1) {
574: err = cudaSetDevice((int)device);
575: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
576: } else {
577: /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */
579: /* First get the device count */
580: err = cudaGetDeviceCount(&devCount);
581: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
583: /* next determine the rank and then set the device via a mod */
584: MPI_Comm_rank(comm,&rank);
585: device = rank % devCount;
586: err = cudaSetDevice((int)device);
587: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
588: }
590: /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
591: multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
592: are ok. */
593: err = cudaSetDeviceFlags(cudaDeviceMapHost);
594: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
595: } else {
596: PetscInt device;
597: cudaError_t err = cudaSuccess;
599: /* the code below works for serial GPU simulations */
600: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
601: if (flg1) {
602: err = cudaSetDevice((int)device);
603: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
604: }
606: /* set the device flags so that it can map host memory ... here, we error check. */
607: err = cudaSetDeviceFlags(cudaDeviceMapHost);
608: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
609: }
611: PetscCUDAInitialized = PETSC_TRUE;
612: }
613: #endif
616: /*
617: Print basic help message
618: */
619: PetscOptionsHasName(NULL,NULL,"-help",&flg1);
620: if (flg1) {
621: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
622: (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
623: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
624: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
625: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
626: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
627: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
628: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
629: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
630: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
631: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
632: (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
633: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
634: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
635: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
636: (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
637: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
638: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
639: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
640: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
641: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
642: (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
643: (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
644: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
645: (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
646: (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
647: (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
648: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
649: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
650: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
651: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
652: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
653: (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
654: #if defined(PETSC_USE_LOG)
655: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
656: (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
657: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
658: #if defined(PETSC_HAVE_MPE)
659: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
660: #endif
661: (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
662: #endif
663: (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
664: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
665: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
666: (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
667: }
669: #if defined(PETSC_HAVE_POPEN)
670: {
671: char machine[128];
672: PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
673: if (flg1) {
674: PetscPOpenSetMachine(machine);
675: }
676: }
677: #endif
679: PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
680: if (flg1) {
681: PetscSleep(si);
682: }
684: PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
685: if (flg1) {
686: PetscStrstr(mname,"null",&f);
687: if (f) {
688: PetscInfoDeactivateClass(0);
689: }
690: }
692: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA)
693: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
694: if (!flg3) {
695: PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
696: }
697: #endif
698: #if defined(PETSC_HAVE_VIENNACL)
699: PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
700: PetscViennaCLSynchronize = flg3;
701: #endif
702: #if defined(PETSC_HAVE_VECCUDA)
703: PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
704: PetscCUDASynchronize = flg3;
705: #endif
707: #if defined(PETSC_HAVE_VIENNACL)
708: PetscViennaCLInit();
709: #endif
711: return(0);
712: }