55 std::string
readopts(
"PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS" );
56 std::string readoptsPhysAtm(
"PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION" );
65 int repartitioner_scheme = 0;
66 #ifdef MOAB_HAVE_ZOLTAN
67 repartitioner_scheme = 2;
70 MPI_Init( &argc, &argv );
77 "../../sandbox/MeshFiles/e3sm/ne4pg2_o240/ne4pg2_p8.h5m";
78 std::string atmPhysMesh =
79 "../../sandbox/MeshFiles/e3sm/ne4pg2_o240/AtmPhys_pg2.h5m";
97 int cmpatm = 5, cplatm = 6;
99 #ifdef ENABLE_ATMOCN_COUPLING
100 std::string ocnFilename = TestDir +
"unittest/recMeshOcn.h5m";
101 int rankInOcnComm = -1;
102 int cmpocn = 17, cplocn = 18, atmocnid = 618,
105 #ifdef ENABLE_ATMLND_COUPLING
106 std::string lndFilename =
"../../sandbox/MeshFiles/e3sm/ne4pg2_o240/land_p8.h5m";
107 int rankInLndComm = -1;
108 int cpllnd = 10, cmplnd = 9, atmlndid = 610,
112 int rankInCouComm = -1;
124 opts.
addOpt< std::string >(
"atmosphere,t",
"atm mesh filename (source)", &
atmFilename );
125 #ifdef ENABLE_ATMOCN_COUPLING
126 opts.
addOpt< std::string >(
"ocean,m",
"ocean mesh filename (target)", &ocnFilename );
129 #ifdef ENABLE_ATMLND_COUPLING
130 opts.
addOpt< std::string >(
"land,l",
"land mesh filename (target)", &lndFilename );
133 opts.
addOpt< std::string >(
"physgrid,q",
"physics grid file", &atmPhysMesh );
135 opts.
addOpt<
int >(
"startAtm,a",
"start task for atmosphere layout", &
startG1 );
136 opts.
addOpt<
int >(
"endAtm,b",
"end task for atmosphere layout", &
endG1 );
137 #ifdef ENABLE_ATMOCN_COUPLING
138 opts.
addOpt<
int >(
"startOcn,c",
"start task for ocean layout", &
startG2 );
139 opts.
addOpt<
int >(
"endOcn,d",
"end task for ocean layout", &
endG2 );
142 #ifdef ENABLE_ATMLND_COUPLING
143 opts.
addOpt<
int >(
"startLnd,e",
"start task for land layout", &startG3 );
144 opts.
addOpt<
int >(
"endLnd,f",
"end task for land layout", &endG3 );
147 opts.
addOpt<
int >(
"startCoupler,g",
"start task for coupler layout", &startG4 );
148 opts.
addOpt<
int >(
"endCoupler,j",
"end task for coupler layout", &endG4 );
150 opts.
addOpt<
int >(
"partitioning,p",
"partitioning option for migration", &repartitioner_scheme );
159 #ifdef ENABLE_ATMOCN_COUPLING
160 "\n ocn file: " << ocnFilename <<
"\n on tasks : " <<
startG2 <<
":" <<
endG2 <<
162 #ifdef ENABLE_ATMLND_COUPLING
163 "\n land file: " << lndFilename <<
"\n on tasks : " << startG3 <<
":" << endG3 <<
166 "\n atm phys file: " << atmPhysMesh <<
"\n on tasks : " <<
startG1 <<
":" <<
endG1 <<
168 "\n partitioning (0 trivial, 1 graph, 2 geometry) " << repartitioner_scheme <<
"\n ";
173 MPI_Group atmPEGroup;
176 CHECKIERR(
ierr,
"Cannot create atm MPI group and communicator " )
178 #ifdef ENABLE_ATMOCN_COUPLING
179 MPI_Group ocnPEGroup;
182 CHECKIERR(
ierr,
"Cannot create ocn MPI group and communicator " )
185 #ifdef ENABLE_ATMLND_COUPLING
186 MPI_Group lndPEGroup;
189 CHECKIERR(
ierr,
"Cannot create lnd MPI group and communicator " )
193 MPI_Group couPEGroup;
196 CHECKIERR(
ierr,
"Cannot create cpl MPI group and communicator " )
202 MPI_Group joinAtmCouGroup;
207 #ifdef ENABLE_ATMOCN_COUPLING
209 MPI_Group joinOcnCouGroup;
212 CHECKIERR(
ierr,
"Cannot create joint ocn cou communicator" )
215 #ifdef ENABLE_ATMLND_COUPLING
217 MPI_Group joinLndCouGroup;
220 CHECKIERR(
ierr,
"Cannot create joint ocn cou communicator" )
226 int cmpAtmAppID = -1;
228 int cplAtmAppID = -1;
230 #ifdef ENABLE_ATMOCN_COUPLING
231 int cmpOcnAppID = -1;
233 int cplOcnAppID = -1, cplAtmOcnAppID = -1, cplOcnAtmAppID = -1;
240 #ifdef ENABLE_ATMLND_COUPLING
241 int cmpLndAppID = -1;
243 int cplLndAppID = -1, cplAtmLndAppID = -1, cplLndAtmAppID = -1;
250 int cmpPhysAtmID = -1;
253 if( couComm != MPI_COMM_NULL )
255 MPI_Comm_rank( couComm, &rankInCouComm );
259 #ifdef ENABLE_ATMOCN_COUPLING
264 #ifdef ENABLE_ATMLND_COUPLING
270 if( atmComm != MPI_COMM_NULL )
277 #ifdef ENABLE_ATMOCN_COUPLING
278 if( ocnComm != MPI_COMM_NULL )
280 MPI_Comm_rank( ocnComm, &rankInOcnComm );
294 #ifdef ENABLE_ATMOCN_COUPLING
298 &couPEGroup, &ocnCouComm, ocnFilename,
readopts,
nghlay, repartitioner_scheme );
301 if( couComm != MPI_COMM_NULL )
303 char outputFileTgt3[] =
"recvOcn3.h5m";
304 PUSH_TIMER(
"Write migrated OCN mesh on coupler PEs" )
313 if( atmComm != MPI_COMM_NULL )
325 int nverts[3], nelem[3];
328 printf( "Phys Atm Component Mesh: %d vertices and %d elements\n", nverts[0], nelem[0] );
333 #ifdef ENABLE_ATMLND_COUPLING
335 if( lndComm != MPI_COMM_NULL )
341 &couPEGroup, &lndCouComm, lndFilename, readoptsPhysAtm,
nghlay,
342 repartitioner_scheme );
344 if( couComm != MPI_COMM_NULL )
346 char outputFileLnd[] =
"recvLnd.h5m";
353 #ifdef ENABLE_ATMOCN_COUPLING
354 if( couComm != MPI_COMM_NULL )
358 CHECKIERR(
ierr,
"Cannot register ocn_atm intx over coupler pes " )
361 CHECKIERR(
ierr, "Cannot register atm_ocn intx over coupler pes " )
365 #ifdef ENABLE_ATMLND_COUPLING
366 if( couComm != MPI_COMM_NULL )
370 CHECKIERR(
ierr,
"Cannot register atm_lnd intx over coupler pes " )
373 CHECKIERR(
ierr, "Cannot register lnd_atm intx over coupler pes " )
377 const char* weights_identifiers[2] = {
"scalar",
"scalar-pc" };
378 int disc_orders[3] = { 4, 1, 1 };
379 const char* disc_methods[3] = {
"cgll",
"fv",
"pcloud" };
380 const char* dof_tag_names[3] = {
"GLOBAL_DOFS",
"GLOBAL_ID",
"GLOBAL_ID" };
381 #ifdef ENABLE_ATMOCN_COUPLING
382 if( couComm != MPI_COMM_NULL )
384 PUSH_TIMER(
"Compute ATM-OCN mesh intersection" )
385 ierr = iMOAB_ComputeMeshIntersectionOnSphere( cplAtmPID, cplOcnPID, cplAtmOcnPID );
394 iMOAB_ComputeMeshIntersectionOnSphere( cplOcnPID, cplAtmPID, cplOcnAtmPID );
399 if( atmCouComm != MPI_COMM_NULL )
407 PUSH_TIMER(
"Compute OCN coverage graph for ATM mesh" )
408 ierr = iMOAB_CoverageGraph( &atmCouComm, cmpAtmPID, cplAtmPID, cplAtmOcnPID, &
cmpatm, &cplatm,
410 CHECKIERR(
ierr, "cannot recompute direct coverage graph for ocean" )
413 if( ocnCouComm != MPI_COMM_NULL )
423 PUSH_TIMER(
"Compute ATM coverage graph for OCN mesh" )
424 ierr = iMOAB_CoverageGraph( &ocnCouComm, cmpOcnPID, cplOcnPID, cplOcnAtmPID, &cmpocn, &cplocn,
426 CHECKIERR(
ierr, "cannot recompute direct coverage graph for atm" )
431 if( atmCouComm != MPI_COMM_NULL )
435 ierr = iMOAB_ComputeCommGraph( cmpPhAtmPID, cplAtmOcnPID, &atmCouComm, &atmPEGroup, &couPEGroup, &
typeA, &
typeB,
436 &cmpPhysAtm, &atmocnid );
437 CHECKIERR(
ierr,
"cannot compute graph between phys grid on atm and intx between FV atm and ocn" )
450 if( atmCouComm != MPI_COMM_NULL )
454 ierr = iMOAB_ComputeCommGraph( cmpPhAtmPID, cplAtmPID, &atmCouComm, &atmPEGroup, &couPEGroup, &
typeA, &
typeB,
455 &cmpPhysAtm, &cplatm );
456 CHECKIERR(
ierr,
"cannot compute graph between phys grid on atm and FV atm on coupler" )
460 #ifdef ENABLE_ATMLND_COUPLING
461 if( couComm != MPI_COMM_NULL )
463 PUSH_TIMER(
"Compute ATM-LND mesh intersection" )
464 ierr = iMOAB_ComputeMeshIntersectionOnSphere( cplAtmPID, cplLndPID, cplAtmLndPID );
465 CHECKIERR(
ierr, "failed to compute atm - land intx for mapping" );
469 ierr = iMOAB_ComputeMeshIntersectionOnSphere( cplLndPID, cplAtmPID, cplLndAtmPID );
473 if( atmCouComm != MPI_COMM_NULL )
481 PUSH_TIMER(
"Compute LND coverage graph for ATM mesh" )
482 ierr = iMOAB_CoverageGraph( &atmCouComm, cmpAtmPID, cplAtmPID, cplAtmLndPID, &
cmpatm, &cplatm,
484 CHECKIERR(
ierr, "cannot recompute direct coverage graph for land" )
490 if( atmCouComm != MPI_COMM_NULL )
494 ierr = iMOAB_ComputeCommGraph( cmpPhAtmPID, cplAtmLndPID, &atmCouComm, &atmPEGroup, &couPEGroup, &
typeA, &
typeB,
495 &cmpPhysAtm, &atmlndid );
496 CHECKIERR(
ierr,
"cannot compute comm graph between atm and atm/lnd intersection" )
500 if( lndCouComm != MPI_COMM_NULL )
510 PUSH_TIMER(
"Compute ATM coverage graph for LND mesh" )
511 ierr = iMOAB_CoverageGraph( &lndCouComm, cmpLndPID, cplLndPID, cplLndAtmPID, &cmplnd, &cpllnd,
513 CHECKIERR(
ierr, "cannot recompute direct coverage graph for atm for intx with land" )
519 int fMonotoneTypeID = 0, fVolumetric = 0, fValidate = 1, fNoConserve = 0, fNoBubble = 1, fInverseDistanceMap = 0;
521 #ifdef ENABLE_ATMOCN_COUPLING
523 if( couComm != MPI_COMM_NULL )
525 char serialWriteOptions[] =
"";
526 std::stringstream outf;
527 outf <<
"intxAtmOcn_" << rankInCouComm <<
".h5m";
528 std::string intxfile = outf.str();
534 if( couComm != MPI_COMM_NULL )
536 PUSH_TIMER(
"Compute the projection weights with TempestRemap" )
537 ierr = iMOAB_ComputeScalarProjectionWeights( cplAtmOcnPID, weights_identifiers[0], disc_methods[1],
539 disc_methods[1], &disc_orders[1],
540 nullptr, &fNoBubble, &fMonotoneTypeID, &fVolumetric,
541 &fInverseDistanceMap, &fNoConserve, &fValidate, dof_tag_names[1],
543 CHECKIERR(
ierr, "cannot compute scalar projection weights" )
548 if( couComm != MPI_COMM_NULL )
550 PUSH_TIMER(
"Compute the projection weights with TempestRemap for ocn - atm map" )
551 ierr = iMOAB_ComputeScalarProjectionWeights( cplOcnAtmPID, weights_identifiers[0], disc_methods[1],
553 disc_methods[1], &disc_orders[1],
554 nullptr, &fNoBubble, &fMonotoneTypeID, &fVolumetric,
555 &fInverseDistanceMap, &fNoConserve, &fValidate, dof_tag_names[1],
557 CHECKIERR(
ierr, "cannot compute scalar projection weights" )
567 #ifdef ENABLE_ATMLND_COUPLING
568 if( couComm != MPI_COMM_NULL )
571 PUSH_TIMER(
"Compute ATM-LND remapping weights" )
572 ierr = iMOAB_ComputeScalarProjectionWeights( cplAtmLndPID, weights_identifiers[0], disc_methods[1],
573 &disc_orders[1], disc_methods[1], &disc_orders[1],
nullptr,
574 &fNoBubble, &fMonotoneTypeID, &fVolumetric, &fInverseDistanceMap,
575 &fNoConserve, &fValidate, dof_tag_names[1], dof_tag_names[1] );
576 CHECKIERR(
ierr, "failed to compute remapping projection weights for ATM-LND scalar non-conservative field" );
580 PUSH_TIMER( "Compute LND-ATM remapping weights" )
581 ierr = iMOAB_ComputeScalarProjectionWeights( cplLndAtmPID, weights_identifiers[0], disc_methods[1],
582 &disc_orders[1], disc_methods[1], &disc_orders[1],
nullptr,
583 &fNoBubble, &fMonotoneTypeID, &fVolumetric, &fInverseDistanceMap,
584 &fNoConserve, &fValidate, dof_tag_names[1], dof_tag_names[1] );
585 CHECKIERR(
ierr, "failed to compute remapping projection weights for LND-ATM scalar non-conservative field" );
592 int atmCompNDoFs = 1 , ocnCompNDoFs = 1 ;
594 const char* bottomFields =
"T_ph:u_ph:v_ph";
596 const char* bottomProjectedFields =
"T_proj:u_proj:v_proj";
599 const char* bottomFields2 =
"T2_ph:u2_ph:v2_ph";
602 const char* bottomFields3 =
"T3_ph:u3_ph:v3_ph";
605 const char* bottomPhProjectedFields =
"Tph_proj:uph_proj:vph_proj";
608 const char* bottomPhLndProjectedFields =
609 "TphL_proj:uphL_proj:vphL_proj";
611 if( couComm != MPI_COMM_NULL )
614 CHECKIERR(
ierr,
"failed to define the field tags T_ph, u_ph, v_ph " );
615 #ifdef ENABLE_ATMOCN_COUPLING
618 CHECKIERR(
ierr,
"failed to define the field tags T_proj, u_proj, v_proj " );
621 #ifdef ENABLE_ATMLND_COUPLING
626 CHECKIERR(
ierr,
"failed to define the field tags T_proj, u_proj, v_proj" );
636 if( cplAtmAppID >= 0 )
638 int nverts[3], nelem[3], nblocks[3], nsbc[3], ndbc[3];
647 int numAllElem = nelem[2];
648 std::vector< double > vals;
649 int storLeng = atmCompNDoFs * numAllElem * 3;
650 vals.resize( storLeng );
651 for(
int k = 0; k < storLeng; k++ )
660 #ifdef ENABLE_ATMOCN_COUPLING
661 PUSH_TIMER(
"Send/receive data from atm component to coupler in ocn context" )
662 if( atmComm != MPI_COMM_NULL )
666 ierr = iMOAB_SendElementTag( cmpPhAtmPID,
"T_ph:u_ph:v_ph", &atmCouComm, &atmocnid );
669 if( couComm != MPI_COMM_NULL )
672 ierr = iMOAB_ReceiveElementTag( cplAtmOcnPID,
"T_ph:u_ph:v_ph", &atmCouComm, &cmpPhysAtm );
673 CHECKIERR(
ierr,
"cannot receive tag values on intx atm ocn" )
678 if( atmComm != MPI_COMM_NULL )
680 ierr = iMOAB_FreeSenderBuffers( cmpPhAtmPID, &atmocnid );
681 CHECKIERR(
ierr,
"cannot free buffers used to resend atm tag towards the coverage mesh" )
692 if( couComm != MPI_COMM_NULL )
694 const char* concat_fieldname =
"T_ph:u_ph:v_ph";
695 const char* concat_fieldnameT =
"T_proj:u_proj:v_proj";
699 PUSH_TIMER(
"Apply Scalar projection weights" )
700 ierr = iMOAB_ApplyScalarProjectionWeights( cplAtmOcnPID, weights_identifiers[0], concat_fieldname,
702 CHECKIERR(
ierr, "failed to compute projection weight application" );
705 char outputFileTgt[] = "fOcnOnCpl3.h5m";
710 if( ocnComm != MPI_COMM_NULL )
715 "failed to define the field tag for receiving back the tag T_proj, u_proj, v_proj on ocn pes" );
721 if( couComm != MPI_COMM_NULL )
724 ierr = iMOAB_SendElementTag( cplOcnPID,
"T_proj:u_proj:v_proj", &ocnCouComm, &context_id );
725 CHECKIERR(
ierr,
"cannot send tag values back to ocean pes" )
729 if( ocnComm != MPI_COMM_NULL )
732 ierr = iMOAB_ReceiveElementTag( cmpOcnPID,
"T_proj:u_proj:v_proj", &ocnCouComm, &context_id );
733 CHECKIERR(
ierr,
"cannot receive tag values from ocean mesh on coupler pes" )
738 if( couComm != MPI_COMM_NULL )
741 ierr = iMOAB_FreeSenderBuffers( cplOcnPID, &context_id );
744 if( ocnComm != MPI_COMM_NULL )
746 char outputFileOcn[] =
"OcnWithProj3.h5m";
754 #ifdef ENABLE_ATMLND_COUPLING
762 PUSH_TIMER(
"Send/receive data from phys comp atm to coupler land, using computed graph" )
763 if( atmComm != MPI_COMM_NULL )
768 ierr = iMOAB_SendElementTag( cmpPhAtmPID,
"T_ph:u_ph:v_ph", &atmCouComm, &atmlndid );
769 CHECKIERR(
ierr,
"cannot send tag values towards cpl on land" )
771 if( couComm != MPI_COMM_NULL )
774 ierr = iMOAB_ReceiveElementTag( cplAtmLndPID,
"T_ph:u_ph:v_ph", &atmCouComm, &cmpPhysAtm );
775 CHECKIERR(
ierr,
"cannot receive tag values on land on coupler, for atm coupling" )
780 if( atmComm != MPI_COMM_NULL )
782 ierr = iMOAB_FreeSenderBuffers( cmpPhAtmPID, &atmlndid );
783 CHECKIERR(
ierr,
"cannot free buffers used to resend atm tag towards the land on coupler" )
786 if( couComm != MPI_COMM_NULL )
788 const char* concat_fieldname =
"T_ph:u_ph:v_ph";
789 const char* concat_fieldnameT =
"T_proj:u_proj:v_proj";
793 PUSH_TIMER(
"Apply Scalar projection weights" )
794 ierr = iMOAB_ApplyScalarProjectionWeights( cplAtmLndPID, weights_identifiers[0], concat_fieldname,
796 CHECKIERR(
ierr, "failed to compute projection weight application" );
799 char outputFileTgt[] = "fLndOnCpl3.h5m";
812 if( lndComm != MPI_COMM_NULL )
817 "failed to define the field tag for receiving back the tag T_proj, u_proj, v_proj on lnd pes" );
819 if( couComm != MPI_COMM_NULL )
822 ierr = iMOAB_SendElementTag( cplLndPID,
"T_proj:u_proj:v_proj", &lndCouComm, &context_id );
826 if( lndComm != MPI_COMM_NULL )
829 ierr = iMOAB_ReceiveElementTag( cmpLndPID,
"T_proj:u_proj:v_proj", &lndCouComm, &context_id );
830 CHECKIERR(
ierr,
"cannot receive tag values from land mesh on coupler pes" )
834 if( couComm != MPI_COMM_NULL )
837 ierr = iMOAB_FreeSenderBuffers( cplLndPID, &context_id );
838 CHECKIERR(
ierr,
"cannot free buffers related to sending tags from coupler to land pes" )
840 if( lndComm != MPI_COMM_NULL )
842 char outputFileLnd[] =
"LndWithProj3.h5m";
851 #ifdef ENABLE_ATMOCN_COUPLING
852 PUSH_TIMER(
"Send/receive data from ocn component to coupler in atm context" )
853 if( ocnComm != MPI_COMM_NULL )
857 ierr = iMOAB_SendElementTag( cmpOcnPID,
"T_proj:u_proj:v_proj", &ocnCouComm, &cplatm );
858 CHECKIERR(
ierr,
"cannot send tag values T_proj, etc towards ocn coupler" )
860 if( couComm != MPI_COMM_NULL )
863 ierr = iMOAB_ReceiveElementTag( cplOcnPID,
"T_proj:u_proj:v_proj", &ocnCouComm, &cplatm );
869 if( ocnComm != MPI_COMM_NULL )
871 ierr = iMOAB_FreeSenderBuffers( cmpOcnPID, &cplatm );
872 CHECKIERR(
ierr,
"cannot free buffers used to send ocn tag towards the coverage mesh for atm" )
875 if( couComm != MPI_COMM_NULL )
878 char outputFileRecvd[] =
"recvOcnCpl.h5m";
880 CHECKIERR(
ierr,
"could not write recvOcnCplOcn.h5m to disk" )
884 if( couComm != MPI_COMM_NULL )
889 const char* concat_fieldname =
"T_proj:u_proj:v_proj";
890 const char* concat_fieldnameT =
"T2_ph:u2_ph:v2_ph";
893 CHECKIERR(
ierr,
"failed to define the field tags T2_ph, u2_ph, v2_ph" );
895 PUSH_TIMER(
"Apply Scalar projection weights" )
896 ierr = iMOAB_ApplyScalarProjectionWeights( cplLndAtmPID, weights_identifiers[0], concat_fieldname,
898 CHECKIERR(
ierr, "failed to compute projection weight application" );
901 char outputFileTgt[] = "fAtm2OnCpl2.h5m";
908 if( atmComm != MPI_COMM_NULL )
912 iMOAB_DefineTagStorage( cmpPhAtmPID, bottomPhProjectedFields, &tagTypes[1], &atmCompNDoFs, &tagIndexIn2 );
913 CHECKIERR(
ierr,
"failed to define the field tag for receiving back the tag "
914 "Tph_proj, etc on atm pes" );
925 if( couComm != MPI_COMM_NULL )
927 context_id = cmpPhysAtm;
928 ierr = iMOAB_SendElementTag( cplAtmPID,
"T2_ph:u2_ph:v2_ph", &atmCouComm, &context_id );
933 if( atmComm != MPI_COMM_NULL )
936 ierr = iMOAB_ReceiveElementTag( cmpPhAtmPID,
"Tph_proj:uph_proj:vph_proj", &atmCouComm, &context_id );
937 CHECKIERR(
ierr,
"cannot receive tag values from atm pg2 mesh on coupler pes" )
942 if( couComm != MPI_COMM_NULL )
945 ierr = iMOAB_FreeSenderBuffers( cplAtmPID, &context_id );
946 CHECKIERR(
ierr,
"cannot free buffers for sending T2_ph from cpl to phys atm" )
948 if( atmComm != MPI_COMM_NULL )
950 char outputFileAtmPh[] =
"AtmPhysProj.h5m";
959 #ifdef ENABLE_ATMLND_COUPLING
960 PUSH_TIMER(
"Send/receive data from lnd component to coupler in atm context" )
961 if( lndComm != MPI_COMM_NULL )
964 ierr = iMOAB_SendElementTag( cmpLndPID,
"T_proj:u_proj:v_proj", &lndCouComm, &cplatm );
965 CHECKIERR(
ierr,
"cannot send tag values T_proj, etc towards lnd coupler" )
967 if( couComm != MPI_COMM_NULL )
970 ierr = iMOAB_ReceiveElementTag( cplLndPID,
"T_proj:u_proj:v_proj", &lndCouComm, &cplatm );
976 if( lndComm != MPI_COMM_NULL )
978 ierr = iMOAB_FreeSenderBuffers( cmpLndPID, &cplatm );
979 CHECKIERR(
ierr,
"cannot free buffers used to send lnd tag towards the coverage mesh for atm" )
982 if( couComm != MPI_COMM_NULL )
985 char outputFileRecvd[] =
"recvLndCpl.h5m";
991 if( couComm != MPI_COMM_NULL )
993 PUSH_TIMER(
"Apply Scalar projection weights for lnd - atm coupling" )
994 const
char* concat_fieldname = "T_proj:u_proj:v_proj";
995 const
char* concat_fieldnameT = "T3_ph:u3_ph:v3_ph";
1000 ierr = iMOAB_ApplyScalarProjectionWeights( cplLndAtmPID, weights_identifiers[0], concat_fieldname,
1001 concat_fieldnameT );
1002 CHECKIERR(
ierr, "failed to compute projection weight application from lnd to atm " );
1005 char outputFileTgt[] = "fAtm3OnCpl.h5m";
1012 if( atmComm != MPI_COMM_NULL )
1017 CHECKIERR(
ierr,
"failed to define the field tag for receiving back the tag "
1018 "TphL_proj, on atm pes" );
1029 if( couComm != MPI_COMM_NULL )
1031 context_id = cmpPhysAtm;
1032 ierr = iMOAB_SendElementTag( cplAtmPID,
"T3_ph:u3_ph:v3_ph", &atmCouComm, &context_id );
1037 if( atmComm != MPI_COMM_NULL )
1039 context_id = cplatm;
1040 ierr = iMOAB_ReceiveElementTag( cmpPhAtmPID,
"TphL_proj:uphL_proj:vphL_proj", &atmCouComm, &context_id );
1041 CHECKIERR(
ierr,
"cannot receive tag values from atm pg2 mesh on coupler pes" )
1046 if( couComm != MPI_COMM_NULL )
1049 ierr = iMOAB_FreeSenderBuffers( cplAtmPID, &context_id );
1050 CHECKIERR(
ierr,
"cannot free buffers used for sending back atm tags " )
1052 if( atmComm != MPI_COMM_NULL )
1054 char outputFileAtmPh[] =
"AtmPhysProj3.h5m";
1056 CHECKIERR(
ierr,
"could not write AtmPhysProj3.h5m to disk" )
1060 if( couComm != MPI_COMM_NULL )
1067 if( couComm != MPI_COMM_NULL )
1074 #ifdef ENABLE_ATMOCN_COUPLING
1075 if( couComm != MPI_COMM_NULL )
1080 if( couComm != MPI_COMM_NULL )
1087 #ifdef ENABLE_ATMLND_COUPLING
1088 if( lndComm != MPI_COMM_NULL )
1094 if( atmComm != MPI_COMM_NULL )
1099 #ifdef ENABLE_ATMOCN_COUPLING
1100 if( ocnComm != MPI_COMM_NULL )
1107 if( atmComm != MPI_COMM_NULL )
1113 #ifdef ENABLE_ATMLND_COUPLING
1114 if( couComm != MPI_COMM_NULL )
1121 #ifdef ENABLE_ATMOCN_COUPLING
1122 if( couComm != MPI_COMM_NULL )
1129 if( couComm != MPI_COMM_NULL )
1140 if( MPI_COMM_NULL != atmCouComm ) MPI_Comm_free( &atmCouComm );
1141 MPI_Group_free( &joinAtmCouGroup );
1142 if( MPI_COMM_NULL != atmComm ) MPI_Comm_free( &atmComm );
1144 #ifdef ENABLE_ATMOCN_COUPLING
1145 if( MPI_COMM_NULL != ocnComm ) MPI_Comm_free( &ocnComm );
1147 if( MPI_COMM_NULL != ocnCouComm ) MPI_Comm_free( &ocnCouComm );
1148 MPI_Group_free( &joinOcnCouGroup );
1151 #ifdef ENABLE_ATMLND_COUPLING
1152 if( MPI_COMM_NULL != lndComm ) MPI_Comm_free( &lndComm );
1154 if( MPI_COMM_NULL != lndCouComm ) MPI_Comm_free( &lndCouComm );
1155 MPI_Group_free( &joinLndCouGroup );
1158 if( MPI_COMM_NULL != couComm ) MPI_Comm_free( &couComm );
1160 MPI_Group_free( &atmPEGroup );
1161 #ifdef ENABLE_ATMOCN_COUPLING
1162 MPI_Group_free( &ocnPEGroup );
1164 #ifdef ENABLE_ATMLND_COUPLING
1165 MPI_Group_free( &lndPEGroup );
1167 MPI_Group_free( &couPEGroup );
1168 MPI_Group_free( &
jgroup );