2: #ifndef __THREADCOMMIMPL_H

  5: #include <petscthreadcomm.h>

  7: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
  8: #include <sys/sysinfo.h>
  9: #endif
 10: #if defined(PETSC_HAVE_UNISTD_H)
 11: #include <unistd.h>
 12: #endif
 13: #if defined(PETSC_HAVE_STDLIB_H)
 14: #include <stdlib.h>
 15: #endif
 16: #if defined(PETSC_HAVE_SYS_SYSCTL_H)
 17: #include <sys/sysctl.h>
 18: #endif
 19: #if defined(PETSC_HAVE_WINDOWS_H)
 20: #include <windows.h>
 21: #endif

 23: PETSC_EXTERN PetscMPIInt Petsc_ThreadComm_keyval;

 25: /* Max. number of arguments for kernel */
 26: #define PETSC_KERNEL_NARGS_MAX 10

 28: /* Max. number of kernels */
 29: #define PETSC_KERNELS_MAX 10

 31: /* Status of threads */
 32: #define THREADCOMM_THREAD_WAITING_FOR_NEWRED 0
 33: #define THREADCOMM_THREAD_POSTED_LOCALRED    1
 34: /* Status of the reduction */
 35: #define THREADCOMM_REDUCTION_NONE           -1
 36: #define THREADCOMM_REDUCTION_NEW             0
 37: #define THREADCOMM_REDUCTION_COMPLETE        1

 39: #define PetscReadOnce(type,val) (*(volatile type *)&val)

 41: struct _p_PetscThreadCommRedCtx{
 42:   PetscThreadComm               tcomm;          /* The associated threadcomm */
 43:   PetscInt                      red_status;     /* Reduction status */
 44:   PetscInt                      nworkThreads;   /* Number of threads doing the reduction */
 45:   PetscInt                      *thread_status; /* Reduction status of each thread */
 46:   void                          *local_red;     /* Array to hold local reduction contribution from each thread */
 47:   PetscThreadCommReductionOp    op;             /* The reduction operation */
 48:   PetscDataType                 type;           /* The reduction data type */
 49: };

 51: typedef struct _p_PetscThreadCommJobCtx *PetscThreadCommJobCtx;
 52: struct  _p_PetscThreadCommJobCtx{
 53:   PetscThreadComm   tcomm;                         /* The thread communicator */
 54:   PetscInt          nargs;                         /* Number of arguments for the kernel */
 55:   PetscThreadKernel pfunc;                         /* Kernel function */
 56:   void              *args[PETSC_KERNEL_NARGS_MAX]; /* Array of void* to hold the arguments */
 57:   PetscScalar       scalars[3];                    /* Array to hold three scalar values */
 58: };

 60: /* Structure to manage job queue */
 61: typedef struct _p_PetscThreadCommJobQueue *PetscThreadCommJobQueue;
 62: struct _p_PetscThreadCommJobQueue{
 63:   PetscInt ctr;                                  /* job counter */
 64:   PetscThreadCommJobCtx jobs[PETSC_KERNELS_MAX]; /* queue of jobs */
 65: };

 67: typedef struct _PetscThreadCommOps *PetscThreadCommOps;
 68: struct _PetscThreadCommOps {
 69:   PetscErrorCode (*destroy)(PetscThreadComm);
 70:   PetscErrorCode (*runkernel)(MPI_Comm,PetscThreadCommJobCtx);
 71:   PetscErrorCode (*view)(PetscThreadComm,PetscViewer);
 72:   PetscErrorCode (*barrier)(PetscThreadComm);
 73:   PetscInt       (*getrank)(void);
 74: };

 76: struct _p_PetscThreadComm{
 77:   PetscInt                refct;
 78:   PetscInt                nworkThreads; /* Number of threads in the pool */
 79:   PetscInt                *affinities;  /* Thread affinity */
 80:   PetscThreadCommOps      ops;          /* Operations table */
 81:   void                    *data;        /* implementation specific data */
 82:   PetscThreadCommJobQueue jobqueue;     /* Job queue */
 83:   char                    type[256];    /* Thread model type */
 84:   PetscInt                leader;       /* Rank of the leader thread. This thread manages
 85:                                            the synchronization for collective operatons like reductions.
 86:                                         */
 87:   PetscThreadCommRedCtx   red;      /* Reduction context */
 88: };

 90: /* register thread communicator models */
 91: PETSC_EXTERN PetscErrorCode PetscThreadCommRegister(const char[],const char[],const char[],PetscErrorCode(*)(PetscThreadComm));
 92: PETSC_EXTERN PetscErrorCode PetscThreadCommRegisterAll(const char path[]);
 93: #if defined(PETSC_USE_DYNAMIC_LIBRARIES)
 94: #define PetscThreadCommRegisterDynamic(a,b,c,d) PetscThreadCommRegister(a,b,c,0)
 95: #else
 96: #define PetscThreadCommRegisterDynamic(a,b,c,d) PetscThreadCommRegister(a,b,c,d)
 97: #endif

 99: PETSC_EXTERN PetscErrorCode PetscThreadCommReductionCreate(PetscThreadComm,PetscThreadCommRedCtx*);
100: PETSC_EXTERN PetscErrorCode PetscThreadCommReductionDestroy(PetscThreadCommRedCtx);
101: PETSC_EXTERN PetscErrorCode PetscRunKernel(PetscInt,PetscInt,PetscThreadCommJobCtx);
102: #endif