Actual source code: init.c
petsc-3.8.4 2018-03-24
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot known
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h>
10: #include <petsc/private/petscimpl.h>
11: #include <petscvalgrind.h>
12: #include <petscviewer.h>
14: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
15: #include <sys/sysinfo.h>
16: #endif
17: #if defined(PETSC_HAVE_UNISTD_H)
18: #include <unistd.h>
19: #endif
20: #if defined(PETSC_HAVE_CUDA)
21: #include <cuda_runtime.h>
22: #endif
24: #if defined(PETSC_HAVE_VIENNACL)
25: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
26: #endif
28: /* ------------------------Nasty global variables -------------------------------*/
29: /*
30: Indicates if PETSc started up MPI, or it was
31: already started before PETSc was initialized.
32: */
33: PetscBool PetscBeganMPI = PETSC_FALSE;
34: PetscBool PetscInitializeCalled = PETSC_FALSE;
35: PetscBool PetscFinalizeCalled = PETSC_FALSE;
36: PetscBool PetscCUDAInitialized = PETSC_FALSE;
38: PetscMPIInt PetscGlobalRank = -1;
39: PetscMPIInt PetscGlobalSize = -1;
41: #if defined(PETSC_HAVE_COMPLEX)
42: #if defined(PETSC_COMPLEX_INSTANTIATE)
43: template <> class std::complex<double>; /* instantiate complex template class */
44: #endif
45: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
46: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
47: MPI_Datatype MPIU_C_COMPLEX;
48: #endif
50: /*MC
51: PETSC_i - the imaginary number i
53: Synopsis:
54: #include <petscsys.h>
55: PetscComplex PETSC_i;
57: Level: beginner
59: Note:
60: Complex numbers are automatically available if PETSc located a working complex implementation
62: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
63: M*/
64: PetscComplex PETSC_i;
65: #endif
66: #if defined(PETSC_USE_REAL___FLOAT128)
67: MPI_Datatype MPIU___FLOAT128 = 0;
68: #if defined(PETSC_HAVE_COMPLEX)
69: MPI_Datatype MPIU___COMPLEX128 = 0;
70: #endif
71: #elif defined(PETSC_USE_REAL___FP16)
72: MPI_Datatype MPIU___FP16 = 0;
73: #endif
74: MPI_Datatype MPIU_2SCALAR = 0;
75: #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
76: MPI_Datatype MPIU_2INT = 0;
77: #endif
78: MPI_Datatype MPIU_BOOL;
79: MPI_Datatype MPIU_ENUM;
81: /*
82: Function that is called to display all error messages
83: */
84: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
85: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
86: #if defined(PETSC_HAVE_MATLAB_ENGINE)
87: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintf_Matlab;
88: #else
89: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
90: #endif
91: /*
92: This is needed to turn on/off GPU synchronization
93: */
94: PetscBool PetscCUSPSynchronize = PETSC_FALSE;
95: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
96: PetscBool PetscCUDASynchronize = PETSC_FALSE;
98: /* ------------------------------------------------------------------------------*/
99: /*
100: Optional file where all PETSc output from various prints is saved
101: */
102: FILE *petsc_history = NULL;
104: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
105: {
107: PetscMPIInt rank,size;
108: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
109: char version[256];
112: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
113: if (!rank) {
114: char arch[10];
115: int err;
117: PetscGetArchType(arch,10);
118: PetscGetDate(date,64);
119: PetscGetVersion(version,256);
120: MPI_Comm_size(PETSC_COMM_WORLD,&size);
121: if (filename) {
122: PetscFixFilename(filename,fname);
123: } else {
124: PetscGetHomeDirectory(pfile,240);
125: PetscStrcat(pfile,"/.petschistory");
126: PetscFixFilename(pfile,fname);
127: }
129: *fd = fopen(fname,"a");
130: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
132: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
133: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
134: PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
135: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
136: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
138: err = fflush(*fd);
139: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
140: }
141: return(0);
142: }
144: PetscErrorCode PetscCloseHistoryFile(FILE **fd)
145: {
147: PetscMPIInt rank;
148: char date[64];
149: int err;
152: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
153: if (!rank) {
154: PetscGetDate(date,64);
155: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
156: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
157: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
158: err = fflush(*fd);
159: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160: err = fclose(*fd);
161: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
162: }
163: return(0);
164: }
166: /* ------------------------------------------------------------------------------*/
168: /*
169: This is ugly and probably belongs somewhere else, but I want to
170: be able to put a true MPI abort error handler with command line args.
172: This is so MPI errors in the debugger will leave all the stack
173: frames. The default MP_Abort() cleans up and exits thus providing no useful information
174: in the debugger hence we call abort() instead of MPI_Abort().
175: */
177: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
178: {
180: (*PetscErrorPrintf)("MPI error %d\n",*flag);
181: abort();
182: }
184: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
185: {
189: (*PetscErrorPrintf)("MPI error %d\n",*flag);
190: PetscAttachDebugger();
191: if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
192: }
194: /*@C
195: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
196: wishes a clean exit somewhere deep in the program.
198: Collective on PETSC_COMM_WORLD
200: Options Database Keys are the same as for PetscFinalize()
202: Level: advanced
204: Note:
205: See PetscInitialize() for more general runtime options.
207: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
208: @*/
209: PetscErrorCode PetscEnd(void)
210: {
212: PetscFinalize();
213: exit(0);
214: return 0;
215: }
217: PetscBool PetscOptionsPublish = PETSC_FALSE;
218: extern PetscErrorCode PetscSetUseTrMalloc_Private(void);
219: extern PetscErrorCode PetscSetUseHBWMalloc_Private(void);
220: extern PetscBool petscsetmallocvisited;
221: static char emacsmachinename[256];
223: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
224: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0;
226: /*@C
227: PetscSetHelpVersionFunctions - Sets functions that print help and version information
228: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
229: This routine enables a "higher-level" package that uses PETSc to print its messages first.
231: Input Parameter:
232: + help - the help function (may be NULL)
233: - version - the version function (may be NULL)
235: Level: developer
237: Concepts: package help message
239: @*/
240: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
241: {
243: PetscExternalHelpFunction = help;
244: PetscExternalVersionFunction = version;
245: return(0);
246: }
248: #if defined(PETSC_USE_LOG)
249: extern PetscBool PetscObjectsLog;
250: #endif
252: PetscErrorCode PetscOptionsCheckInitial_Private(void)
253: {
254: char string[64],mname[PETSC_MAX_PATH_LEN],*f;
255: MPI_Comm comm = PETSC_COMM_WORLD;
256: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
257: PetscErrorCode ierr;
258: PetscReal si;
259: PetscInt intensity;
260: int i;
261: PetscMPIInt rank;
262: char version[256],helpoptions[256];
263: #if !defined(PETSC_HAVE_THREADSAFETY)
264: PetscReal logthreshold;
265: #endif
266: #if defined(PETSC_USE_LOG)
267: PetscViewerFormat format;
268: PetscBool flg4 = PETSC_FALSE;
269: #endif
272: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
274: #if !defined(PETSC_HAVE_THREADSAFETY)
275: /*
276: Setup the memory management; support for tracing malloc() usage
277: */
278: PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
279: logthreshold = 0.0;
280: PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
281: if (flg1) flg3 = PETSC_TRUE;
282: #if defined(PETSC_USE_DEBUG)
283: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
284: if ((!flg2 || flg1) && !petscsetmallocvisited) {
285: if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
286: /* turn off default -malloc if valgrind is being used */
287: PetscSetUseTrMalloc_Private();
288: }
289: }
290: #else
291: PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
292: PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
293: if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
294: #endif
295: if (flg3) {
296: PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
297: }
298: flg1 = PETSC_FALSE;
299: PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
300: if (flg1) {
301: PetscSetUseTrMalloc_Private();
302: PetscMallocDebug(PETSC_TRUE);
303: }
304: flg1 = PETSC_FALSE;
305: PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
306: #if defined(PETSC_USE_DEBUG)
307: if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
308: PetscSetUseTrMalloc_Private();
309: PetscMallocSetDumpLog();
310: PetscMallocDebug(PETSC_TRUE);
311: }
312: #endif
313: flg1 = PETSC_FALSE;
314: PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
315: if (flg1) {PetscSetUseHBWMalloc_Private();}
317: flg1 = PETSC_FALSE;
318: PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
319: if (!flg1) {
320: flg1 = PETSC_FALSE;
321: PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
322: }
323: if (flg1) {
324: PetscMemorySetGetMaximumUsage();
325: }
326: #endif
328: #if defined(PETSC_USE_LOG)
329: PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
330: #endif
332: /*
333: Set the display variable for graphics
334: */
335: PetscSetDisplay();
337: /*
338: Print the PETSc version information
339: */
340: PetscOptionsHasName(NULL,NULL,"-v",&flg1);
341: PetscOptionsHasName(NULL,NULL,"-version",&flg2);
342: PetscOptionsHasName(NULL,NULL,"-help",&flg3);
343: if (flg1 || flg2 || flg3) {
345: /*
346: Print "higher-level" package version message
347: */
348: if (PetscExternalVersionFunction) {
349: (*PetscExternalVersionFunction)(comm);
350: }
352: PetscGetVersion(version,256);
353: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
354: (*PetscHelpPrintf)(comm,"%s\n",version);
355: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
356: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
357: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
358: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
359: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
360: (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
361: }
363: /*
364: Print "higher-level" package help message
365: */
366: if (flg3) {
367: if (PetscExternalHelpFunction) {
368: (*PetscExternalHelpFunction)(comm);
369: }
370: }
372: PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
373: if (flg1) {
374: PetscStrcmp(helpoptions,"intro",&flg2);
375: if (flg2) {
376: PetscOptionsDestroyDefault();
377: PetscFreeMPIResources();
378: MPI_Finalize();
379: exit(0);
380: }
381: }
383: /*
384: Setup the error handling
385: */
386: flg1 = PETSC_FALSE;
387: PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
388: if (flg1) {
389: MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_ARE_FATAL);
390: PetscPushErrorHandler(PetscAbortErrorHandler,0);
391: }
392: flg1 = PETSC_FALSE;
393: PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
394: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
395: flg1 = PETSC_FALSE;
396: PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
397: if (flg1) {
398: MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
399: }
400: flg1 = PETSC_FALSE;
401: PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
402: if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
403: flg1 = PETSC_FALSE;
404: PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
405: if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
406: PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
409: /*
410: Setup debugger information
411: */
412: PetscSetDefaultDebugger();
413: PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
414: if (flg1) {
415: MPI_Errhandler err_handler;
417: PetscSetDebuggerFromString(string);
418: MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
419: MPI_Comm_set_errhandler(comm,err_handler);
420: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
421: }
422: PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
423: if (flg1) { PetscSetDebugTerminal(string); }
424: PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
425: PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
426: if (flg1 || flg2) {
427: PetscMPIInt size;
428: PetscInt lsize,*nodes;
429: MPI_Errhandler err_handler;
430: /*
431: we have to make sure that all processors have opened
432: connections to all other processors, otherwise once the
433: debugger has stated it is likely to receive a SIGUSR1
434: and kill the program.
435: */
436: MPI_Comm_size(PETSC_COMM_WORLD,&size);
437: if (size > 2) {
438: PetscMPIInt dummy = 0;
439: MPI_Status status;
440: for (i=0; i<size; i++) {
441: if (rank != i) {
442: MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
443: }
444: }
445: for (i=0; i<size; i++) {
446: if (rank != i) {
447: MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
448: }
449: }
450: }
451: /* check if this processor node should be in debugger */
452: PetscMalloc1(size,&nodes);
453: lsize = size;
454: PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
455: if (flag) {
456: for (i=0; i<lsize; i++) {
457: if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
458: }
459: }
460: if (!flag) {
461: PetscSetDebuggerFromString(string);
462: PetscPushErrorHandler(PetscAbortErrorHandler,0);
463: if (flg1) {
464: PetscAttachDebugger();
465: } else {
466: PetscStopForDebugger();
467: }
468: MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
469: MPI_Comm_set_errhandler(comm,err_handler);
470: }
471: PetscFree(nodes);
472: }
474: PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
475: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
477: /*
478: Setup profiling and logging
479: */
480: #if defined(PETSC_USE_INFO)
481: {
482: char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
483: PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
484: if (flg1 && logname[0]) {
485: PetscInfoAllow(PETSC_TRUE,logname);
486: } else if (flg1) {
487: PetscInfoAllow(PETSC_TRUE,NULL);
488: }
489: }
490: #endif
491: #if defined(PETSC_USE_LOG)
492: mname[0] = 0;
493: PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
494: if (flg1) {
495: if (mname[0]) {
496: PetscOpenHistoryFile(mname,&petsc_history);
497: } else {
498: PetscOpenHistoryFile(NULL,&petsc_history);
499: }
500: }
501: #if defined(PETSC_HAVE_MPE)
502: flg1 = PETSC_FALSE;
503: PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
504: if (flg1) {PetscLogMPEBegin();}
505: #endif
506: flg1 = PETSC_FALSE;
507: flg3 = PETSC_FALSE;
508: PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
509: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
510: if (flg1) { PetscLogAllBegin(); }
511: else if (flg3) { PetscLogDefaultBegin();}
513: PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
514: if (flg1) {
515: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
516: FILE *file;
517: if (mname[0]) {
518: PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
519: PetscFixFilename(name,fname);
520: file = fopen(fname,"w");
521: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
522: } else file = PETSC_STDOUT;
523: PetscLogTraceBegin(file);
524: }
526: PetscOptionsGetViewer(PETSC_COMM_WORLD,NULL,"-log_view",NULL,&format,&flg4);
527: if (flg4) {
528: if (format == PETSC_VIEWER_ASCII_XML){
529: PetscLogNestedBegin();
530: } else {
531: PetscLogDefaultBegin();
532: }
533: }
534: #endif
536: PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
538: #if defined(PETSC_HAVE_CUDA)
539: PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
540: if (flg1) {
541: struct cudaDeviceProp prop;
542: int devCount;
543: PetscInt device;
544: cudaError_t err = cudaSuccess;
546: err = cudaGetDeviceCount(&devCount);
547: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
548: for (device = 0; device < devCount; ++device) {
549: err = cudaGetDeviceProperties(&prop, (int)device);
550: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
551: PetscPrintf(PETSC_COMM_WORLD, "CUDA device %D: %s\n", device, prop.name);
552: }
553: }
554: if (!PetscCUDAInitialized) {
555: PetscMPIInt size;
556: MPI_Comm_size(PETSC_COMM_WORLD,&size);
557: if (size>1) {
558: int devCount;
559: PetscInt device;
560: PetscMPIInt rank;
561: cudaError_t err = cudaSuccess;
563: /* check to see if we force multiple ranks to hit the same GPU */
564: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
565: if (flg1) {
566: err = cudaSetDevice((int)device);
567: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
568: } else {
569: /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */
571: /* First get the device count */
572: err = cudaGetDeviceCount(&devCount);
573: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
575: /* next determine the rank and then set the device via a mod */
576: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
577: device = rank % devCount;
578: err = cudaSetDevice((int)device);
579: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
580: }
582: /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
583: multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
584: are ok. */
585: err = cudaSetDeviceFlags(cudaDeviceMapHost);
586: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
587: } else {
588: PetscInt device;
589: cudaError_t err = cudaSuccess;
591: /* the code below works for serial GPU simulations */
592: PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
593: if (flg1) {
594: err = cudaSetDevice((int)device);
595: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
596: }
598: /* set the device flags so that it can map host memory ... here, we error check. */
599: err = cudaSetDeviceFlags(cudaDeviceMapHost);
600: if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
601: }
603: PetscCUDAInitialized = PETSC_TRUE;
604: }
605: #endif
608: /*
609: Print basic help message
610: */
611: PetscOptionsHasName(NULL,NULL,"-help",&flg1);
612: if (flg1) {
613: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
614: (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
615: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
616: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
617: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
618: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
619: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
620: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
621: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
622: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
623: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
624: (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
625: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
626: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
627: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
628: (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
629: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
630: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
631: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
632: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
633: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
634: (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
635: (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
636: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
637: (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
638: (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
639: (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
640: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
641: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
642: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
643: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
644: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
645: (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
646: #if defined(PETSC_USE_LOG)
647: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
648: (*PetscHelpPrintf)(comm," -log[_summary _summary_python]: logging objects and events\n");
649: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
650: #if defined(PETSC_HAVE_MPE)
651: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
652: #endif
653: (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
654: #endif
655: (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
656: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
657: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
658: (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
659: }
661: #if defined(PETSC_HAVE_POPEN)
662: {
663: char machine[128];
664: PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
665: if (flg1) {
666: PetscPOpenSetMachine(machine);
667: }
668: }
669: #endif
671: PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
672: if (flg1) {
673: PetscSleep(si);
674: }
676: PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
677: if (flg1) {
678: PetscStrstr(mname,"null",&f);
679: if (f) {
680: PetscInfoDeactivateClass(0);
681: }
682: }
684: #if defined(PETSC_HAVE_CUSP) || defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA)
685: PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
686: if (!flg3) {
687: PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
688: }
689: #endif
690: #if defined(PETSC_HAVE_CUSP)
691: PetscOptionsGetBool(NULL,NULL,"-cusp_synchronize",&flg3,NULL);
692: PetscCUSPSynchronize = flg3;
693: #elif defined(PETSC_HAVE_VIENNACL)
694: PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
695: PetscViennaCLSynchronize = flg3;
696: #elif defined(PETSC_HAVE_VECCUDA)
697: PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
698: PetscCUDASynchronize = flg3;
699: #endif
701: #if defined(PETSC_HAVE_VIENNACL)
702: PetscViennaCLInit();
703: #endif
705: return(0);
706: }