Actual source code: init.c

  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot know 
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>        /*I  "petscsys.h"   I*/

 11: #ifndef _GNU_SOURCE
 12: #define _GNU_SOURCE
 13: #endif
 14: #if defined(PETSC_HAVE_SCHED_H)
 15: #ifndef __USE_GNU
 17: #endif
 18: #include <sched.h>
 19: #endif
 20: #if defined(PETSC_HAVE_PTHREAD_H)
 21: #include <pthread.h>
 22: #endif

 24: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 25: #include <sys/sysinfo.h>
 26: #endif
 27: #if defined(PETSC_HAVE_UNISTD_H)
 28: #include <unistd.h>
 29: #endif
 30: #if defined(PETSC_HAVE_STDLIB_H)
 31: #include <stdlib.h>
 32: #endif
 33: #if defined(PETSC_HAVE_MALLOC_H)
 34: #include <malloc.h>
 35: #endif
 36: #if defined(PETSC_HAVE_VALGRIND)
 37: #include <valgrind/valgrind.h>
 38: #endif

 40: /* ------------------------Nasty global variables -------------------------------*/
 41: /*
 42:      Indicates if PETSc started up MPI, or it was 
 43:    already started before PETSc was initialized.
 44: */
 45: PetscBool    PetscBeganMPI         = PETSC_FALSE;
 46: PetscBool    PetscInitializeCalled = PETSC_FALSE;
 47: PetscBool    PetscFinalizeCalled   = PETSC_FALSE;
 48: PetscBool    PetscUseThreadPool    = PETSC_FALSE;
 49: PetscBool    PetscThreadGo         = PETSC_TRUE;
 50: PetscMPIInt  PetscGlobalRank = -1;
 51: PetscMPIInt  PetscGlobalSize = -1;

 53: #if defined(PETSC_HAVE_PTHREADCLASSES)
 54: PetscMPIInt  PetscMaxThreads = 2;
 55: pthread_t*   PetscThreadPoint;
 56: #define PETSC_HAVE_PTHREAD_BARRIER
 57: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
 58: pthread_barrier_t* BarrPoint;   /* used by 'true' thread pool */
 59: #endif
 60: PetscErrorCode ithreaderr = 0;
 61: int*         pVal;

 63: #define CACHE_LINE_SIZE 64  /* used by 'chain', 'main','tree' thread pools */
 64: int* ThreadCoreAffinity;

 66: typedef enum {JobInitiated,ThreadsWorking,JobCompleted} estat;  /* used by 'chain','tree' thread pool */

 68: typedef struct {
 69:   pthread_mutex_t** mutexarray;
 70:   pthread_cond_t**  cond1array;
 71:   pthread_cond_t** cond2array;
 72:   void* (*pfunc)(void*);
 73:   void** pdata;
 74:   PetscBool startJob;
 75:   estat eJobStat;
 76:   PetscBool** arrThreadStarted;
 77:   PetscBool** arrThreadReady;
 78: } sjob_tree;
 79: sjob_tree job_tree;
 80: typedef struct {
 81:   pthread_mutex_t** mutexarray;
 82:   pthread_cond_t**  cond1array;
 83:   pthread_cond_t** cond2array;
 84:   void* (*pfunc)(void*);
 85:   void** pdata;
 86:   PetscBool** arrThreadReady;
 87: } sjob_main;
 88: sjob_main job_main;
 89: typedef struct {
 90:   pthread_mutex_t** mutexarray;
 91:   pthread_cond_t**  cond1array;
 92:   pthread_cond_t** cond2array;
 93:   void* (*pfunc)(void*);
 94:   void** pdata;
 95:   PetscBool startJob;
 96:   estat eJobStat;
 97:   PetscBool** arrThreadStarted;
 98:   PetscBool** arrThreadReady;
 99: } sjob_chain;
100: sjob_chain job_chain;
101: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
102: typedef struct {
103:   pthread_mutex_t mutex;
104:   pthread_cond_t cond;
105:   void* (*pfunc)(void*);
106:   void** pdata;
107:   pthread_barrier_t* pbarr;
108:   int iNumJobThreads;
109:   int iNumReadyThreads;
110:   PetscBool startJob;
111: } sjob_true;
112: sjob_true job_true = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE};
113: #endif

115: pthread_cond_t  main_cond  = PTHREAD_COND_INITIALIZER;  /* used by 'true', 'chain','tree' thread pools */
116: char* arrmutex; /* used by 'chain','main','tree' thread pools */
117: char* arrcond1; /* used by 'chain','main','tree' thread pools */
118: char* arrcond2; /* used by 'chain','main','tree' thread pools */
119: char* arrstart; /* used by 'chain','main','tree' thread pools */
120: char* arrready; /* used by 'chain','main','tree' thread pools */

122: /* Function Pointers */
123: void*          (*PetscThreadFunc)(void*) = NULL;
124: void*          (*PetscThreadInitialize)(PetscInt) = NULL;
125: PetscErrorCode (*PetscThreadFinalize)(void) = NULL;
126: void           (*MainWait)(void) = NULL;
127: PetscErrorCode (*MainJob)(void* (*pFunc)(void*),void**,PetscInt) = NULL;
128: /**** Tree Thread Pool Functions ****/
129: void*          PetscThreadFunc_Tree(void*);
130: void*          PetscThreadInitialize_Tree(PetscInt);
131: PetscErrorCode PetscThreadFinalize_Tree(void);
132: void           MainWait_Tree(void);
133: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void**,PetscInt);
134: /**** Main Thread Pool Functions ****/
135: void*          PetscThreadFunc_Main(void*);
136: void*          PetscThreadInitialize_Main(PetscInt);
137: PetscErrorCode PetscThreadFinalize_Main(void);
138: void           MainWait_Main(void);
139: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void**,PetscInt);
140: /**** Chain Thread Pool Functions ****/
141: void*          PetscThreadFunc_Chain(void*);
142: void*          PetscThreadInitialize_Chain(PetscInt);
143: PetscErrorCode PetscThreadFinalize_Chain(void);
144: void           MainWait_Chain(void);
145: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void**,PetscInt);
146: /**** True Thread Pool Functions ****/
147: void*          PetscThreadFunc_True(void*);
148: void*          PetscThreadInitialize_True(PetscInt);
149: PetscErrorCode PetscThreadFinalize_True(void);
150: void           MainWait_True(void);
151: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void**,PetscInt);
152: /**** NO Thread Pool Function  ****/
153: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void**,PetscInt);
154: /****  ****/
155: void* FuncFinish(void*);
156: void* PetscThreadRun(MPI_Comm Comm,void* (*pFunc)(void*),int,pthread_t*,void**);
157: void* PetscThreadStop(MPI_Comm Comm,int,pthread_t*);
158: #endif

160: #if defined(PETSC_USE_COMPLEX)
161: #if defined(PETSC_COMPLEX_INSTANTIATE)
162: template <> class std::complex<double>; /* instantiate complex template class */
163: #endif
164: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
165: MPI_Datatype   MPIU_C_DOUBLE_COMPLEX;
166: MPI_Datatype   MPIU_C_COMPLEX;
167: #endif
168: PetscScalar    PETSC_i;
169: #else
170: PetscScalar    PETSC_i = 0.0;
171: #endif
172: #if defined(PETSC_USE_REAL___FLOAT128)
173: MPI_Datatype   MPIU___FLOAT128 = 0;
174: #endif
175: MPI_Datatype   MPIU_2SCALAR = 0;
176: MPI_Datatype   MPIU_2INT = 0;

178: /*
179:      These are needed by petscbt.h
180: */
181: #include <petscbt.h>
182: char      _BT_mask = ' ';
183: char      _BT_c = ' ';
184: PetscInt  _BT_idx  = 0;

186: /*
187:        Function that is called to display all error messages
188: */
189: PetscErrorCode  (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
190: PetscErrorCode  (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
191: #if defined(PETSC_HAVE_MATLAB_ENGINE)
192: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintf_Matlab;
193: #else
194: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
195: #endif
196: /*
197:   This is needed to turn on/off cusp synchronization */
198: PetscBool   synchronizeCUSP = PETSC_FALSE;

200: /* ------------------------------------------------------------------------------*/
201: /* 
202:    Optional file where all PETSc output from various prints is saved
203: */
204: FILE *petsc_history = PETSC_NULL;

208: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
209: {
211:   PetscMPIInt    rank,size;
212:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
213:   char           version[256];

216:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
217:   if (!rank) {
218:     char        arch[10];
219:     int         err;
220:     PetscViewer viewer;

222:     PetscGetArchType(arch,10);
223:     PetscGetDate(date,64);
224:     PetscGetVersion(version,256);
225:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
226:     if (filename) {
227:       PetscFixFilename(filename,fname);
228:     } else {
229:       PetscGetHomeDirectory(pfile,240);
230:       PetscStrcat(pfile,"/.petschistory");
231:       PetscFixFilename(pfile,fname);
232:     }

234:     *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
235:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
236:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
237:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
238:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
239:     PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);
240:     PetscOptionsView(viewer);
241:     PetscViewerDestroy(&viewer);
242:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
243:     err = fflush(*fd);
244:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
245:   }
246:   return(0);
247: }

251: PetscErrorCode  PetscCloseHistoryFile(FILE **fd)
252: {
254:   PetscMPIInt    rank;
255:   char           date[64];
256:   int            err;

259:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
260:   if (!rank) {
261:     PetscGetDate(date,64);
262:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
263:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
264:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
265:     err = fflush(*fd);
266:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
267:     err = fclose(*fd);
268:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
269:   }
270:   return(0);
271: }

273: /* ------------------------------------------------------------------------------*/

275: /* 
276:    This is ugly and probably belongs somewhere else, but I want to 
277:   be able to put a true MPI abort error handler with command line args.

279:     This is so MPI errors in the debugger will leave all the stack 
280:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
281:   in the debugger hence we call abort() instead of MPI_Abort().
282: */

286: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
287: {
289:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
290:   abort();
291: }

295: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
296: {

300:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
301:   PetscAttachDebugger();
302:   if (ierr) { /* hopeless so get out */
303:     MPI_Abort(*comm,*flag);
304:   }
305: }

309: /*@C 
310:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 
311:      wishes a clean exit somewhere deep in the program.

313:    Collective on PETSC_COMM_WORLD

315:    Options Database Keys are the same as for PetscFinalize()

317:    Level: advanced

319:    Note:
320:    See PetscInitialize() for more general runtime options.

322: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
323: @*/
324: PetscErrorCode  PetscEnd(void)
325: {
327:   PetscFinalize();
328:   exit(0);
329:   return 0;
330: }

332: PetscBool    PetscOptionsPublish = PETSC_FALSE;
335: static char       emacsmachinename[256];

337: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
338: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

342: /*@C 
343:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
344:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
345:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

347:    Input Parameter:
348: +  help - the help function (may be PETSC_NULL)
349: -  version - the version function (may be PETSC_NULL)

351:    Level: developer

353:    Concepts: package help message

355: @*/
356: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
357: {
359:   PetscExternalHelpFunction    = help;
360:   PetscExternalVersionFunction = version;
361:   return(0);
362: }

366: PetscErrorCode  PetscOptionsCheckInitial_Private(void)
367: {
368:   char           string[64],mname[PETSC_MAX_PATH_LEN],*f;
369:   MPI_Comm       comm = PETSC_COMM_WORLD;
370:   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout;
372:   PetscReal      si;
373:   int            i;
374:   PetscMPIInt    rank;
375:   char           version[256];

378:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);

380:   /*
381:       Setup the memory management; support for tracing malloc() usage 
382:   */
383:   PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);
384: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
385:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);
386:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
387: #if defined(PETSC_HAVE_VALGRIND)
388:     if (flg2 || !(RUNNING_ON_VALGRIND)) {
389:       /* turn off default -malloc if valgrind is being used */
390: #endif
391:       PetscSetUseTrMalloc_Private();
392: #if defined(PETSC_HAVE_VALGRIND)
393:     }
394: #endif
395:   }
396: #else
397:   PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);
398:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);
399:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
400: #endif
401:   if (flg3) {
402:     PetscMallocSetDumpLog();
403:   }
404:   flg1 = PETSC_FALSE;
405:   PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);
406:   if (flg1) {
407:     PetscSetUseTrMalloc_Private();
408:     PetscMallocDebug(PETSC_TRUE);
409:   }

411:   flg1 = PETSC_FALSE;
412:   PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);
413:   if (!flg1) {
414:     flg1 = PETSC_FALSE;
415:     PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);
416:   }
417:   if (flg1) {
418:     PetscMemorySetGetMaximumUsage();
419:   }

421:   /*
422:       Set the display variable for graphics
423:   */
424:   PetscSetDisplay();


427:   /*
428:       Print the PETSc version information
429:   */
430:   PetscOptionsHasName(PETSC_NULL,"-v",&flg1);
431:   PetscOptionsHasName(PETSC_NULL,"-version",&flg2);
432:   PetscOptionsHasName(PETSC_NULL,"-help",&flg3);
433:   if (flg1 || flg2 || flg3){

435:     /*
436:        Print "higher-level" package version message 
437:     */
438:     if (PetscExternalVersionFunction) {
439:       (*PetscExternalVersionFunction)(comm);
440:     }

442:     PetscGetVersion(version,256);
443:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
444: ------------------------------\n");
445:     (*PetscHelpPrintf)(comm,"%s\n",version);
446:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
447:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
448:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
449:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
450:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
451:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
452: ------------------------------\n");
453:   }

455:   /*
456:        Print "higher-level" package help message 
457:   */
458:   if (flg3){
459:     if (PetscExternalHelpFunction) {
460:       (*PetscExternalHelpFunction)(comm);
461:     }
462:   }

464:   /*
465:       Setup the error handling
466:   */
467:   flg1 = PETSC_FALSE;
468:   PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);
469:   if (flg1) { PetscPushErrorHandler(PetscAbortErrorHandler,0);}
470:   flg1 = PETSC_FALSE;
471:   PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);
472:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
473:   flg1 = PETSC_FALSE;
474:   PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);
475:   if (flg1) {
476:     MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);
477:   }
478:   flg1 = PETSC_FALSE;
479:   PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
480:   if (!flg1) {PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);}
481:   flg1 = PETSC_FALSE;
482:   PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);
483:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}

485:   /*
486:       Setup debugger information
487:   */
488:   PetscSetDefaultDebugger();
489:   PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);
490:   if (flg1) {
491:     MPI_Errhandler err_handler;

493:     PetscSetDebuggerFromString(string);
494:     MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
495:     MPI_Errhandler_set(comm,err_handler);
496:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
497:   }
498:   PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);
499:   if (flg1) { PetscSetDebugTerminal(string); }
500:   PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);
501:   PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);
502:   if (flg1 || flg2) {
503:     PetscMPIInt    size;
504:     PetscInt       lsize,*nodes;
505:     MPI_Errhandler err_handler;
506:     /*
507:        we have to make sure that all processors have opened 
508:        connections to all other processors, otherwise once the 
509:        debugger has stated it is likely to receive a SIGUSR1
510:        and kill the program. 
511:     */
512:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
513:     if (size > 2) {
514:       PetscMPIInt dummy = 0;
515:       MPI_Status  status;
516:       for (i=0; i<size; i++) {
517:         if (rank != i) {
518:           MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
519:         }
520:       }
521:       for (i=0; i<size; i++) {
522:         if (rank != i) {
523:           MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
524:         }
525:       }
526:     }
527:     /* check if this processor node should be in debugger */
528:     PetscMalloc(size*sizeof(PetscInt),&nodes);
529:     lsize = size;
530:     PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);
531:     if (flag) {
532:       for (i=0; i<lsize; i++) {
533:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
534:       }
535:     }
536:     if (!flag) {
537:       PetscSetDebuggerFromString(string);
538:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
539:       if (flg1) {
540:         PetscAttachDebugger();
541:       } else {
542:         PetscStopForDebugger();
543:       }
544:       MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
545:       MPI_Errhandler_set(comm,err_handler);
546:     }
547:     PetscFree(nodes);
548:   }

550:   PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
551:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

553: #if defined(PETSC_USE_SOCKET_VIEWER)
554:   /*
555:     Activates new sockets for zope if needed
556:   */
557:   PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);
558:   PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);
559:   if (flgz){
560:     int  sockfd;
561:     char hostname[256];
562:     char username[256];
563:     int  remoteport = 9999;

565:     PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);
566:     if (!hostname[0]){
567:       PetscGetHostName(hostname,256);
568:     }
569:     PetscOpenSocket(hostname, remoteport, &sockfd);
570:     PetscGetUserName(username, 256);
571:     PETSC_ZOPEFD = fdopen(sockfd, "w");
572:     if (flgzout){
573:       PETSC_STDOUT = PETSC_ZOPEFD;
574:       fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username);
575:       fprintf(PETSC_STDOUT, "<<<start>>>");
576:     } else {
577:       fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username);
578:       fprintf(PETSC_ZOPEFD, "<<<start>>>");
579:     }
580:   }
581: #endif
582: #if defined(PETSC_USE_SERVER)
583:   PetscOptionsHasName(PETSC_NULL,"-server", &flgz);
584:   if (flgz){
585:     PetscInt port = PETSC_DECIDE;
586:     PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);
587:     PetscWebServe(PETSC_COMM_WORLD,(int)port);
588:   }
589: #endif

591:   /*
592:         Setup profiling and logging
593:   */
594: #if defined (PETSC_USE_INFO)
595:   {
596:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
597:     PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);
598:     if (flg1 && logname[0]) {
599:       PetscInfoAllow(PETSC_TRUE,logname);
600:     } else if (flg1) {
601:       PetscInfoAllow(PETSC_TRUE,PETSC_NULL);
602:     }
603:   }
604: #endif
605: #if defined(PETSC_USE_LOG)
606:   mname[0] = 0;
607:   PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
608:   if (flg1) {
609:     if (mname[0]) {
610:       PetscOpenHistoryFile(mname,&petsc_history);
611:     } else {
612:       PetscOpenHistoryFile(0,&petsc_history);
613:     }
614:   }
615: #if defined(PETSC_HAVE_MPE)
616:   flg1 = PETSC_FALSE;
617:   PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);
618:   if (flg1) PetscLogMPEBegin();
619: #endif
620:   flg1 = PETSC_FALSE;
621:   flg2 = PETSC_FALSE;
622:   flg3 = PETSC_FALSE;
623:   PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);
624:   PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);
625:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
626:   PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);
627:   if (flg1)                      {  PetscLogAllBegin(); }
628:   else if (flg2 || flg3 || flg4) {  PetscLogBegin();}
629: 
630:   PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);
631:   if (flg1) {
632:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
633:     FILE *file;
634:     if (mname[0]) {
635:       sprintf(name,"%s.%d",mname,rank);
636:       PetscFixFilename(name,fname);
637:       file = fopen(fname,"w");
638:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
639:     } else {
640:       file = PETSC_STDOUT;
641:     }
642:     PetscLogTraceBegin(file);
643:   }
644: #endif

646:   /*
647:       Setup building of stack frames for all function calls
648:   */
649: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
650:   PetscStackCreate();
651: #endif

653:   PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);

655: #if defined(PETSC_HAVE_PTHREADCLASSES)
656:   /*
657:       Determine whether user specified maximum number of threads
658:    */
659:   PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);

661:   PetscOptionsHasName(PETSC_NULL,"-main",&flg1);
662:   if(flg1) {
663:     cpu_set_t mset;
664:     int icorr,ncorr = get_nprocs();
665:     PetscOptionsGetInt(PETSC_NULL,"-main",&icorr,PETSC_NULL);
666:     CPU_ZERO(&mset);
667:     CPU_SET(icorr%ncorr,&mset);
668:     sched_setaffinity(0,sizeof(cpu_set_t),&mset);
669:   }

671:   PetscInt N_CORES = get_nprocs();
672:   ThreadCoreAffinity = (int*)malloc(N_CORES*sizeof(int));
673:   char tstr[9];
674:   char tbuf[2];
675:   strcpy(tstr,"-thread");
676:   for(i=0;i<PetscMaxThreads;i++) {
677:     ThreadCoreAffinity[i] = i;
678:     sprintf(tbuf,"%d",i);
679:     strcat(tstr,tbuf);
680:     PetscOptionsHasName(PETSC_NULL,tstr,&flg1);
681:     if(flg1) {
682:       PetscOptionsGetInt(PETSC_NULL,tstr,&ThreadCoreAffinity[i],PETSC_NULL);
683:       ThreadCoreAffinity[i] = ThreadCoreAffinity[i]%N_CORES; /* check on the user */
684:     }
685:     tstr[7] = '\0';
686:   }

688:   /*
689:       Determine whether to use thread pool
690:    */
691:   PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);
692:   if (flg1) {
693:     PetscUseThreadPool = PETSC_TRUE;
694:     /* get the thread pool type */
695:     PetscInt ipool = 0;
696:     const char *choices[4] = {"true","tree","main","chain"};

698:     PetscOptionsGetEList(PETSC_NULL,"-use_thread_pool",choices,4,&ipool,PETSC_NULL);
699:     switch(ipool) {
700:     case 1:
701:       PetscThreadFunc       = &PetscThreadFunc_Tree;
702:       PetscThreadInitialize = &PetscThreadInitialize_Tree;
703:       PetscThreadFinalize   = &PetscThreadFinalize_Tree;
704:       MainWait              = &MainWait_Tree;
705:       MainJob               = &MainJob_Tree;
706:       PetscInfo(PETSC_NULL,"Using tree thread pool\n");
707:       break;
708:     case 2:
709:       PetscThreadFunc       = &PetscThreadFunc_Main;
710:       PetscThreadInitialize = &PetscThreadInitialize_Main;
711:       PetscThreadFinalize   = &PetscThreadFinalize_Main;
712:       MainWait              = &MainWait_Main;
713:       MainJob               = &MainJob_Main;
714:       PetscInfo(PETSC_NULL,"Using main thread pool\n");
715:       break;
716: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
717:     case 3:
718: #else
719:     default:
720: #endif
721:       PetscThreadFunc       = &PetscThreadFunc_Chain;
722:       PetscThreadInitialize = &PetscThreadInitialize_Chain;
723:       PetscThreadFinalize   = &PetscThreadFinalize_Chain;
724:       MainWait              = &MainWait_Chain;
725:       MainJob               = &MainJob_Chain;
726:       PetscInfo(PETSC_NULL,"Using chain thread pool\n");
727:       break;
728: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
729:     default:
730:       PetscThreadFunc       = &PetscThreadFunc_True;
731:       PetscThreadInitialize = &PetscThreadInitialize_True;
732:       PetscThreadFinalize   = &PetscThreadFinalize_True;
733:       MainWait              = &MainWait_True;
734:       MainJob               = &MainJob_True;
735:       PetscInfo(PETSC_NULL,"Using true thread pool\n");
736:       break;
737: #endif
738:     }
739:     PetscThreadInitialize(PetscMaxThreads);
740:   } else {
741:     //need to define these in the case on 'no threads' or 'thread create/destroy'
742:     //could take any of the above versions
743:     MainJob               = &MainJob_Spawn;
744:   }
745: #endif
746:   /*
747:        Print basic help message
748:   */
749:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
750:   if (flg1) {
751:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
752:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
753:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
754:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
755:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
756:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
757:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
758:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
759:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
760:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
761:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
762:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
763:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
764:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
765:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
766:     (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");
767:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
768:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
769:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
770:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
771:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
772:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
773:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
774:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
775:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
776:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
777:     (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
778:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
779:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
780:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
781:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
782:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
783:     (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");
784:     (*PetscHelpPrintf)(comm," -server <port>: Run PETSc webserver (default port is 8080) see PetscWebServe()\n");
785: #if defined(PETSC_USE_LOG)
786:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
787:     (*PetscHelpPrintf)(comm," -log[_all _summary _summary_python]: logging objects and events\n");
788:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
789: #if defined(PETSC_HAVE_MPE)
790:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");
791: #endif
792:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
793: #endif
794:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
795:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
796:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
797:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
798:   }

800:   PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);
801:   if (flg1) {
802:     PetscSleep(si);
803:   }

805:   PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
806:   PetscStrstr(mname,"null",&f);
807:   if (f) {
808:     PetscInfoDeactivateClass(PETSC_NULL);
809:   }

811: #if defined(PETSC_HAVE_CUSP)
812:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
813:   if (flg3) flg1 = PETSC_TRUE;
814:   else flg1 = PETSC_FALSE;
815:   PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);
816:   if (flg1) synchronizeCUSP = PETSC_TRUE;
817: #endif

819:   return(0);
820: }

822: #if defined(PETSC_HAVE_PTHREADCLASSES)

824: /**** 'Tree' Thread Pool Functions ****/
825: void* PetscThreadFunc_Tree(void* arg) {
826:   PetscErrorCode iterr;
827:   int icorr,ierr;
828:   int* pId = (int*)arg;
829:   int ThreadId = *pId,Mary = 2,i,SubWorker;
830:   PetscBool PeeOn;
831:   cpu_set_t mset;
832:   //printf("Thread %d In Tree Thread Function\n",ThreadId);
833:   icorr = ThreadCoreAffinity[ThreadId];
834:   CPU_ZERO(&mset);
835:   CPU_SET(icorr,&mset);
836:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

838:   if((Mary*ThreadId+1)>(PetscMaxThreads-1)) {
839:     PeeOn = PETSC_TRUE;
840:   }
841:   else {
842:     PeeOn = PETSC_FALSE;
843:   }
844:   if(PeeOn==PETSC_FALSE) {
845:     /* check your subordinates, wait for them to be ready */
846:     for(i=1;i<=Mary;i++) {
847:       SubWorker = Mary*ThreadId+i;
848:       if(SubWorker<PetscMaxThreads) {
849:         pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
850:         while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE) {
851:           /* upon entry, automically releases the lock and blocks
852:            upon return, has the lock */
853:           pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
854:         }
855:         pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
856:       }
857:     }
858:     /* your subordinates are now ready */
859:   }
860:   pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
861:   /* update your ready status */
862:   *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
863:   if(ThreadId==0) {
864:     job_tree.eJobStat = JobCompleted;
865:     /* ignal main */
866:     pthread_cond_signal(&main_cond);
867:   }
868:   else {
869:     /* tell your boss that you're ready to work */
870:     pthread_cond_signal(job_tree.cond1array[ThreadId]);
871:   }
872:   /* the while loop needs to have an exit
873:   the 'main' thread can terminate all the threads by performing a broadcast
874:    and calling FuncFinish */
875:   while(PetscThreadGo) {
876:     /*need to check the condition to ensure we don't have to wait
877:       waiting when you don't have to causes problems
878:      also need to check the condition to ensure proper handling of spurious wakeups */
879:     while(*(job_tree.arrThreadReady[ThreadId])==PETSC_TRUE) {
880:       /* upon entry, automically releases the lock and blocks
881:        upon return, has the lock */
882:         pthread_cond_wait(job_tree.cond2array[ThreadId],job_tree.mutexarray[ThreadId]);
883:         *(job_tree.arrThreadStarted[ThreadId]) = PETSC_TRUE;
884:         *(job_tree.arrThreadReady[ThreadId])   = PETSC_FALSE;
885:     }
886:     if(ThreadId==0) {
887:       job_tree.startJob = PETSC_FALSE;
888:       job_tree.eJobStat = ThreadsWorking;
889:     }
890:     pthread_mutex_unlock(job_tree.mutexarray[ThreadId]);
891:     if(PeeOn==PETSC_FALSE) {
892:       /* tell your subordinates it's time to get to work */
893:       for(i=1; i<=Mary; i++) {
894:         SubWorker = Mary*ThreadId+i;
895:         if(SubWorker<PetscMaxThreads) {
896:           pthread_cond_signal(job_tree.cond2array[SubWorker]);
897:         }
898:       }
899:     }
900:     /* do your job */
901:     if(job_tree.pdata==NULL) {
902:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata);
903:     }
904:     else {
905:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata[ThreadId]);
906:     }
907:     if(iterr!=0) {
908:       ithreaderr = 1;
909:     }
910:     if(PetscThreadGo) {
911:       /* reset job, get ready for more */
912:       if(PeeOn==PETSC_FALSE) {
913:         /* check your subordinates, waiting for them to be ready
914:          how do you know for a fact that a given subordinate has actually started? */
915:         for(i=1;i<=Mary;i++) {
916:           SubWorker = Mary*ThreadId+i;
917:           if(SubWorker<PetscMaxThreads) {
918:             pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
919:             while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_tree.arrThreadStarted[SubWorker])==PETSC_FALSE) {
920:               /* upon entry, automically releases the lock and blocks
921:                upon return, has the lock */
922:               pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
923:             }
924:             pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
925:           }
926:         }
927:         /* your subordinates are now ready */
928:       }
929:       pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
930:       *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
931:       if(ThreadId==0) {
932:         job_tree.eJobStat = JobCompleted; /* oot thread: last thread to complete, guaranteed! */
933:         /* root thread signals 'main' */
934:         pthread_cond_signal(&main_cond);
935:       }
936:       else {
937:         /* signal your boss before you go to sleep */
938:         pthread_cond_signal(job_tree.cond1array[ThreadId]);
939:       }
940:     }
941:   }
942:   return NULL;
943: }

947: void* PetscThreadInitialize_Tree(PetscInt N) {
948:   PetscInt i,ierr;
949:   int status;

951:   if(PetscUseThreadPool) {
952:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
953:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
954:     arrmutex = (char*)memalign(Val1,Val2);
955:     arrcond1 = (char*)memalign(Val1,Val2);
956:     arrcond2 = (char*)memalign(Val1,Val2);
957:     arrstart = (char*)memalign(Val1,Val2);
958:     arrready = (char*)memalign(Val1,Val2);
959:     job_tree.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
960:     job_tree.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
961:     job_tree.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
962:     job_tree.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
963:     job_tree.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
964:     /* initialize job structure */
965:     for(i=0; i<PetscMaxThreads; i++) {
966:       job_tree.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
967:       job_tree.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
968:       job_tree.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
969:       job_tree.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
970:       job_tree.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
971:     }
972:     for(i=0; i<PetscMaxThreads; i++) {
973:       pthread_mutex_init(job_tree.mutexarray[i],NULL);
974:       pthread_cond_init(job_tree.cond1array[i],NULL);
975:       pthread_cond_init(job_tree.cond2array[i],NULL);
976:       *(job_tree.arrThreadStarted[i])  = PETSC_FALSE;
977:       *(job_tree.arrThreadReady[i])    = PETSC_FALSE;
978:     }
979:     job_tree.pfunc = NULL;
980:     job_tree.pdata = (void**)malloc(N*sizeof(void*));
981:     job_tree.startJob = PETSC_FALSE;
982:     job_tree.eJobStat = JobInitiated;
983:     pVal = (int*)malloc(N*sizeof(int));
984:     /* allocate memory in the heap for the thread structure */
985:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
986:     /* create threads */
987:     for(i=0; i<N; i++) {
988:       pVal[i] = i;
989:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
990:       /* should check status */
991:     }
992:   }
993:   return NULL;
994: }

998: PetscErrorCode PetscThreadFinalize_Tree() {
999:   int i,ierr;
1000:   void* jstatus;


1004:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1005:   /* join the threads */
1006:   for(i=0; i<PetscMaxThreads; i++) {
1007:     pthread_join(PetscThreadPoint[i],&jstatus);
1008:     /* do error checking*/
1009:   }
1010:   free(PetscThreadPoint);
1011:   free(arrmutex);
1012:   free(arrcond1);
1013:   free(arrcond2);
1014:   free(arrstart);
1015:   free(arrready);
1016:   free(job_tree.pdata);
1017:   free(pVal);

1019:   return(0);
1020: }

1024: void MainWait_Tree() {
1025:   int ierr;
1026:   pthread_mutex_lock(job_tree.mutexarray[0]);
1027:   while(job_tree.eJobStat<JobCompleted||job_tree.startJob==PETSC_TRUE) {
1028:     pthread_cond_wait(&main_cond,job_tree.mutexarray[0]);
1029:   }
1030:   pthread_mutex_unlock(job_tree.mutexarray[0]);
1031: }

1035: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void** data,PetscInt n) {
1036:   int i,ierr;
1037:   PetscErrorCode ijoberr = 0;

1039:   MainWait();
1040:   job_tree.pfunc = pFunc;
1041:   job_tree.pdata = data;
1042:   job_tree.startJob = PETSC_TRUE;
1043:   for(i=0; i<PetscMaxThreads; i++) {
1044:     *(job_tree.arrThreadStarted[i]) = PETSC_FALSE;
1045:   }
1046:   job_tree.eJobStat = JobInitiated;
1047:   pthread_cond_signal(job_tree.cond2array[0]);
1048:   if(pFunc!=FuncFinish) {
1049:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1050:   }

1052:   if(ithreaderr) {
1053:     ijoberr = ithreaderr;
1054:   }
1055:   return ijoberr;
1056: }
1057: /****  ****/

1059: /**** 'Main' Thread Pool Functions ****/
1060: void* PetscThreadFunc_Main(void* arg) {
1061:   PetscErrorCode iterr;
1062:   int icorr,ierr;
1063:   int* pId = (int*)arg;
1064:   int ThreadId = *pId;
1065:   cpu_set_t mset;
1066:   //printf("Thread %d In Main Thread Function\n",ThreadId);
1067:   icorr = ThreadCoreAffinity[ThreadId];
1068:   CPU_ZERO(&mset);
1069:   CPU_SET(icorr,&mset);
1070:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1072:   pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1073:   /* update your ready status */
1074:   *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1075:   /* tell the BOSS that you're ready to work before you go to sleep */
1076:   pthread_cond_signal(job_main.cond1array[ThreadId]);

1078:   /* the while loop needs to have an exit
1079:      the 'main' thread can terminate all the threads by performing a broadcast
1080:      and calling FuncFinish */
1081:   while(PetscThreadGo) {
1082:     /* need to check the condition to ensure we don't have to wait
1083:        waiting when you don't have to causes problems
1084:      also need to check the condition to ensure proper handling of spurious wakeups */
1085:     while(*(job_main.arrThreadReady[ThreadId])==PETSC_TRUE) {
1086:       /* upon entry, atomically releases the lock and blocks
1087:        upon return, has the lock */
1088:         pthread_cond_wait(job_main.cond2array[ThreadId],job_main.mutexarray[ThreadId]);
1089:         /* (job_main.arrThreadReady[ThreadId])   = PETSC_FALSE; */
1090:     }
1091:     pthread_mutex_unlock(job_main.mutexarray[ThreadId]);
1092:     if(job_main.pdata==NULL) {
1093:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata);
1094:     }
1095:     else {
1096:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata[ThreadId]);
1097:     }
1098:     if(iterr!=0) {
1099:       ithreaderr = 1;
1100:     }
1101:     if(PetscThreadGo) {
1102:       /* reset job, get ready for more */
1103:       pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1104:       *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1105:       /* tell the BOSS that you're ready to work before you go to sleep */
1106:       pthread_cond_signal(job_main.cond1array[ThreadId]);
1107:     }
1108:   }
1109:   return NULL;
1110: }

1114: void* PetscThreadInitialize_Main(PetscInt N) {
1115:   PetscInt i,ierr;
1116:   int status;

1118:   if(PetscUseThreadPool) {
1119:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1120:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1121:     arrmutex = (char*)memalign(Val1,Val2);
1122:     arrcond1 = (char*)memalign(Val1,Val2);
1123:     arrcond2 = (char*)memalign(Val1,Val2);
1124:     arrstart = (char*)memalign(Val1,Val2);
1125:     arrready = (char*)memalign(Val1,Val2);
1126:     job_main.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1127:     job_main.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1128:     job_main.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1129:     job_main.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1130:     /* initialize job structure */
1131:     for(i=0; i<PetscMaxThreads; i++) {
1132:       job_main.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1133:       job_main.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1134:       job_main.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1135:       job_main.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1136:     }
1137:     for(i=0; i<PetscMaxThreads; i++) {
1138:       pthread_mutex_init(job_main.mutexarray[i],NULL);
1139:       pthread_cond_init(job_main.cond1array[i],NULL);
1140:       pthread_cond_init(job_main.cond2array[i],NULL);
1141:       *(job_main.arrThreadReady[i])    = PETSC_FALSE;
1142:     }
1143:     job_main.pfunc = NULL;
1144:     job_main.pdata = (void**)malloc(N*sizeof(void*));
1145:     pVal = (int*)malloc(N*sizeof(int));
1146:     /* allocate memory in the heap for the thread structure */
1147:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1148:     /* create threads */
1149:     for(i=0; i<N; i++) {
1150:       pVal[i] = i;
1151:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1152:       /* error check */
1153:     }
1154:   }
1155:   else {
1156:   }
1157:   return NULL;
1158: }

1162: PetscErrorCode PetscThreadFinalize_Main() {
1163:   int i,ierr;
1164:   void* jstatus;


1168:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1169:   /* join the threads */
1170:   for(i=0; i<PetscMaxThreads; i++) {
1171:     pthread_join(PetscThreadPoint[i],&jstatus);
1172:   }
1173:   free(PetscThreadPoint);
1174:   free(arrmutex);
1175:   free(arrcond1);
1176:   free(arrcond2);
1177:   free(arrstart);
1178:   free(arrready);
1179:   free(job_main.pdata);
1180:   free(pVal);

1182:   return(0);
1183: }

1187: void MainWait_Main() {
1188:   int i,ierr;
1189:   for(i=0; i<PetscMaxThreads; i++) {
1190:     pthread_mutex_lock(job_main.mutexarray[i]);
1191:     while(*(job_main.arrThreadReady[i])==PETSC_FALSE) {
1192:       pthread_cond_wait(job_main.cond1array[i],job_main.mutexarray[i]);
1193:     }
1194:     pthread_mutex_unlock(job_main.mutexarray[i]);
1195:   }
1196: }

1200: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void** data,PetscInt n) {
1201:   int i,ierr;
1202:   PetscErrorCode ijoberr = 0;

1204:   MainWait(); /* you know everyone is waiting to be signalled! */
1205:   job_main.pfunc = pFunc;
1206:   job_main.pdata = data;
1207:   for(i=0; i<PetscMaxThreads; i++) {
1208:     *(job_main.arrThreadReady[i]) = PETSC_FALSE; /* why do this?  suppose you get into MainWait first */
1209:   }
1210:   /* tell the threads to go to work */
1211:   for(i=0; i<PetscMaxThreads; i++) {
1212:     pthread_cond_signal(job_main.cond2array[i]);
1213:   }
1214:   if(pFunc!=FuncFinish) {
1215:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1216:   }

1218:   if(ithreaderr) {
1219:     ijoberr = ithreaderr;
1220:   }
1221:   return ijoberr;
1222: }
1223: /****  ****/

1225: /**** Chain Thread Functions ****/
1226: void* PetscThreadFunc_Chain(void* arg) {
1227:   PetscErrorCode iterr;
1228:   int icorr,ierr;
1229:   int* pId = (int*)arg;
1230:   int ThreadId = *pId;
1231:   int SubWorker = ThreadId + 1;
1232:   PetscBool PeeOn;
1233:   cpu_set_t mset;
1234:   //printf("Thread %d In Chain Thread Function\n",ThreadId);
1235:   icorr = ThreadCoreAffinity[ThreadId];
1236:   CPU_ZERO(&mset);
1237:   CPU_SET(icorr,&mset);
1238:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1240:   if(ThreadId==(PetscMaxThreads-1)) {
1241:     PeeOn = PETSC_TRUE;
1242:   }
1243:   else {
1244:     PeeOn = PETSC_FALSE;
1245:   }
1246:   if(PeeOn==PETSC_FALSE) {
1247:     /* check your subordinate, wait for him to be ready */
1248:     pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1249:     while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE) {
1250:       /* upon entry, automically releases the lock and blocks
1251:        upon return, has the lock */
1252:       pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1253:     }
1254:     pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1255:     /* your subordinate is now ready*/
1256:   }
1257:   pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1258:   /* update your ready status */
1259:   *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1260:   if(ThreadId==0) {
1261:     job_chain.eJobStat = JobCompleted;
1262:     /* signal main */
1263:     pthread_cond_signal(&main_cond);
1264:   }
1265:   else {
1266:     /* tell your boss that you're ready to work */
1267:     pthread_cond_signal(job_chain.cond1array[ThreadId]);
1268:   }
1269:   /*  the while loop needs to have an exit
1270:      the 'main' thread can terminate all the threads by performing a broadcast
1271:    and calling FuncFinish */
1272:   while(PetscThreadGo) {
1273:     /* need to check the condition to ensure we don't have to wait
1274:        waiting when you don't have to causes problems
1275:      also need to check the condition to ensure proper handling of spurious wakeups */
1276:     while(*(job_chain.arrThreadReady[ThreadId])==PETSC_TRUE) {
1277:       /*upon entry, automically releases the lock and blocks
1278:        upon return, has the lock */
1279:         pthread_cond_wait(job_chain.cond2array[ThreadId],job_chain.mutexarray[ThreadId]);
1280:         *(job_chain.arrThreadStarted[ThreadId]) = PETSC_TRUE;
1281:         *(job_chain.arrThreadReady[ThreadId])   = PETSC_FALSE;
1282:     }
1283:     if(ThreadId==0) {
1284:       job_chain.startJob = PETSC_FALSE;
1285:       job_chain.eJobStat = ThreadsWorking;
1286:     }
1287:     pthread_mutex_unlock(job_chain.mutexarray[ThreadId]);
1288:     if(PeeOn==PETSC_FALSE) {
1289:       /* tell your subworker it's time to get to work */
1290:       pthread_cond_signal(job_chain.cond2array[SubWorker]);
1291:     }
1292:     /* do your job */
1293:     if(job_chain.pdata==NULL) {
1294:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata);
1295:     }
1296:     else {
1297:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata[ThreadId]);
1298:     }
1299:     if(iterr!=0) {
1300:       ithreaderr = 1;
1301:     }
1302:     if(PetscThreadGo) {
1303:       /* reset job, get ready for more */
1304:       if(PeeOn==PETSC_FALSE) {
1305:         /* check your subordinate, wait for him to be ready
1306:          how do you know for a fact that your subordinate has actually started? */
1307:         pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1308:         while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_chain.arrThreadStarted[SubWorker])==PETSC_FALSE) {
1309:           /* upon entry, automically releases the lock and blocks
1310:            upon return, has the lock */
1311:           pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1312:         }
1313:         pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1314:         /* your subordinate is now ready */
1315:       }
1316:       pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1317:       *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1318:       if(ThreadId==0) {
1319:         job_chain.eJobStat = JobCompleted; /* foreman: last thread to complete, guaranteed! */
1320:         /* root thread (foreman) signals 'main' */
1321:         pthread_cond_signal(&main_cond);
1322:       }
1323:       else {
1324:         /* signal your boss before you go to sleep */
1325:         pthread_cond_signal(job_chain.cond1array[ThreadId]);
1326:       }
1327:     }
1328:   }
1329:   return NULL;
1330: }

1334: void* PetscThreadInitialize_Chain(PetscInt N) {
1335:   PetscInt i,ierr;
1336:   int status;

1338:   if(PetscUseThreadPool) {
1339:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1340:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1341:     arrmutex = (char*)memalign(Val1,Val2);
1342:     arrcond1 = (char*)memalign(Val1,Val2);
1343:     arrcond2 = (char*)memalign(Val1,Val2);
1344:     arrstart = (char*)memalign(Val1,Val2);
1345:     arrready = (char*)memalign(Val1,Val2);
1346:     job_chain.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1347:     job_chain.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1348:     job_chain.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1349:     job_chain.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1350:     job_chain.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1351:     /* initialize job structure */
1352:     for(i=0; i<PetscMaxThreads; i++) {
1353:       job_chain.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1354:       job_chain.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1355:       job_chain.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1356:       job_chain.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
1357:       job_chain.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1358:     }
1359:     for(i=0; i<PetscMaxThreads; i++) {
1360:       pthread_mutex_init(job_chain.mutexarray[i],NULL);
1361:       pthread_cond_init(job_chain.cond1array[i],NULL);
1362:       pthread_cond_init(job_chain.cond2array[i],NULL);
1363:       *(job_chain.arrThreadStarted[i])  = PETSC_FALSE;
1364:       *(job_chain.arrThreadReady[i])    = PETSC_FALSE;
1365:     }
1366:     job_chain.pfunc = NULL;
1367:     job_chain.pdata = (void**)malloc(N*sizeof(void*));
1368:     job_chain.startJob = PETSC_FALSE;
1369:     job_chain.eJobStat = JobInitiated;
1370:     pVal = (int*)malloc(N*sizeof(int));
1371:     /* allocate memory in the heap for the thread structure */
1372:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1373:     /* create threads */
1374:     for(i=0; i<N; i++) {
1375:       pVal[i] = i;
1376:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1377:       /* should check error */
1378:     }
1379:   }
1380:   else {
1381:   }
1382:   return NULL;
1383: }


1388: PetscErrorCode PetscThreadFinalize_Chain() {
1389:   int i,ierr;
1390:   void* jstatus;


1394:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1395:   /* join the threads */
1396:   for(i=0; i<PetscMaxThreads; i++) {
1397:     pthread_join(PetscThreadPoint[i],&jstatus);
1398:     /* should check error */
1399:   }
1400:   free(PetscThreadPoint);
1401:   free(arrmutex);
1402:   free(arrcond1);
1403:   free(arrcond2);
1404:   free(arrstart);
1405:   free(arrready);
1406:   free(job_chain.pdata);
1407:   free(pVal);

1409:   return(0);
1410: }

1414: void MainWait_Chain() {
1415:   int ierr;
1416:   pthread_mutex_lock(job_chain.mutexarray[0]);
1417:   while(job_chain.eJobStat<JobCompleted||job_chain.startJob==PETSC_TRUE) {
1418:     pthread_cond_wait(&main_cond,job_chain.mutexarray[0]);
1419:   }
1420:   pthread_mutex_unlock(job_chain.mutexarray[0]);
1421: }

1425: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void** data,PetscInt n) {
1426:   int i,ierr;
1427:   PetscErrorCode ijoberr = 0;

1429:   MainWait();
1430:   job_chain.pfunc = pFunc;
1431:   job_chain.pdata = data;
1432:   job_chain.startJob = PETSC_TRUE;
1433:   for(i=0; i<PetscMaxThreads; i++) {
1434:     *(job_chain.arrThreadStarted[i]) = PETSC_FALSE;
1435:   }
1436:   job_chain.eJobStat = JobInitiated;
1437:   pthread_cond_signal(job_chain.cond2array[0]);
1438:   if(pFunc!=FuncFinish) {
1439:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1440:   }

1442:   if(ithreaderr) {
1443:     ijoberr = ithreaderr;
1444:   }
1445:   return ijoberr;
1446: }
1447: /****  ****/

1449: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
1450: /**** True Thread Functions ****/
1451: void* PetscThreadFunc_True(void* arg) {
1452:   int icorr,ierr,iVal;
1453:   int* pId = (int*)arg;
1454:   int ThreadId = *pId;
1455:   PetscErrorCode iterr;
1456:   cpu_set_t mset;
1457:   //printf("Thread %d In True Pool Thread Function\n",ThreadId);
1458:   icorr = ThreadCoreAffinity[ThreadId];
1459:   CPU_ZERO(&mset);
1460:   CPU_SET(icorr,&mset);
1461:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1463:   pthread_mutex_lock(&job_true.mutex);
1464:   job_true.iNumReadyThreads++;
1465:   if(job_true.iNumReadyThreads==PetscMaxThreads) {
1466:     pthread_cond_signal(&main_cond);
1467:   }
1468:   /*the while loop needs to have an exit
1469:     the 'main' thread can terminate all the threads by performing a broadcast
1470:    and calling FuncFinish */
1471:   while(PetscThreadGo) {
1472:     /*need to check the condition to ensure we don't have to wait
1473:       waiting when you don't have to causes problems
1474:      also need to wait if another thread sneaks in and messes with the predicate */
1475:     while(job_true.startJob==PETSC_FALSE&&job_true.iNumJobThreads==0) {
1476:       /* upon entry, automically releases the lock and blocks
1477:        upon return, has the lock */
1478:       //printf("Thread %d Going to Sleep!\n",ThreadId);
1479:       pthread_cond_wait(&job_true.cond,&job_true.mutex);
1480:     }
1481:     job_true.startJob = PETSC_FALSE;
1482:     job_true.iNumJobThreads--;
1483:     job_true.iNumReadyThreads--;
1484:     iVal = PetscMaxThreads-job_true.iNumReadyThreads-1;
1485:     pthread_mutex_unlock(&job_true.mutex);
1486:     if(job_true.pdata==NULL) {
1487:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata);
1488:     }
1489:     else {
1490:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata[iVal]);
1491:     }
1492:     if(iterr!=0) {
1493:       ithreaderr = 1;
1494:     }
1495:     //printf("Thread %d Finished Job\n",ThreadId);
1496:     /* the barrier is necessary BECAUSE: look at job_true.iNumReadyThreads
1497:       what happens if a thread finishes before they all start? BAD!
1498:      what happens if a thread finishes before any else start? BAD! */
1499:     pthread_barrier_wait(job_true.pbarr); /* ensures all threads are finished */
1500:     /* reset job */
1501:     if(PetscThreadGo) {
1502:       pthread_mutex_lock(&job_true.mutex);
1503:       job_true.iNumReadyThreads++;
1504:       if(job_true.iNumReadyThreads==PetscMaxThreads) {
1505:         /* signal the 'main' thread that the job is done! (only done once) */
1506:         pthread_cond_signal(&main_cond);
1507:       }
1508:     }
1509:   }
1510:   return NULL;
1511: }

1515: void* PetscThreadInitialize_True(PetscInt N) {
1516:   PetscInt i;
1517:   int status;

1519:   pVal = (int*)malloc(N*sizeof(int));
1520:   /* allocate memory in the heap for the thread structure */
1521:   PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1522:   BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); /* BarrPoint[0] makes no sense, don't use it! */
1523:   job_true.pdata = (void**)malloc(N*sizeof(void*));
1524:   for(i=0; i<N; i++) {
1525:     pVal[i] = i;
1526:     status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1527:     /* error check to ensure proper thread creation */
1528:     status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1);
1529:     /* should check error */
1530:   }
1531:   //printf("Finished True Thread Pool Initialization\n");
1532:   return NULL;
1533: }


1538: PetscErrorCode PetscThreadFinalize_True() {
1539:   int i,ierr;
1540:   void* jstatus;


1544:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1545:   /* join the threads */
1546:   for(i=0; i<PetscMaxThreads; i++) {
1547:     pthread_join(PetscThreadPoint[i],&jstatus);
1548:   }
1549:   free(BarrPoint);
1550:   free(PetscThreadPoint);

1552:   return(0);
1553: }

1557: void MainWait_True() {
1558:   int ierr;
1559:   pthread_mutex_lock(&job_true.mutex);
1560:   while(job_true.iNumReadyThreads<PetscMaxThreads||job_true.startJob==PETSC_TRUE) {
1561:     pthread_cond_wait(&main_cond,&job_true.mutex);
1562:   }
1563:   pthread_mutex_unlock(&job_true.mutex);
1564: }

1568: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void** data,PetscInt n) {
1569:   int ierr;
1570:   PetscErrorCode ijoberr = 0;

1572:   MainWait();
1573:   job_true.pfunc = pFunc;
1574:   job_true.pdata = data;
1575:   job_true.pbarr = &BarrPoint[n];
1576:   job_true.iNumJobThreads = n;
1577:   job_true.startJob = PETSC_TRUE;
1578:   pthread_cond_broadcast(&job_true.cond);
1579:   if(pFunc!=FuncFinish) {
1580:     MainWait(); /* why wait after? guarantees that job gets done */
1581:   }

1583:   if(ithreaderr) {
1584:     ijoberr = ithreaderr;
1585:   }
1586:   return ijoberr;
1587: }
1588: /**** NO THREAD POOL FUNCTION ****/
1591: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void** data,PetscInt n) {
1592:   PetscErrorCode ijoberr = 0;

1594:   pthread_t* apThread = (pthread_t*)malloc(n*sizeof(pthread_t));
1595:   PetscThreadPoint = apThread; /* point to same place */
1596:   PetscThreadRun(MPI_COMM_WORLD,pFunc,n,apThread,data);
1597:   PetscThreadStop(MPI_COMM_WORLD,n,apThread); /* ensures that all threads are finished with the job */
1598:   free(apThread);

1600:   return ijoberr;
1601: }
1602: /****  ****/
1603: #endif

1605: void* FuncFinish(void* arg) {
1606:   PetscThreadGo = PETSC_FALSE;
1607:   return(0);
1608: }

1610: #endif