53 std::string
readopts(
"PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS" );
54 std::string readoptsPhysAtm(
"PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION" );
61 int repartitioner_scheme = 0;
62 #ifdef MOAB_HAVE_ZOLTAN
63 repartitioner_scheme = 2;
66 MPI_Init( &argc, &argv );
72 std::string
atmFilename = TestDir +
"unittest/wholeATM_T.h5m";
73 std::string atmPhysMesh =
74 TestDir +
"unittest/AtmPhys_01.h5m";
92 #ifdef ENABLE_ATMOCN_COUPLING
93 std::string ocnFilename = TestDir +
"unittest/recMeshOcn.h5m";
94 int rankInOcnComm = -1;
95 int cmpocn = 17, cplocn = 18,
98 #ifdef ENABLE_ATMLND_COUPLING
99 std::string lndFilename = TestDir +
"unittest/wholeLnd.h5m";
104 int rankInCouComm = -1;
116 opts.
addOpt< std::string >(
"atmosphere,t",
"atm mesh filename (source)", &
atmFilename );
117 #ifdef ENABLE_ATMOCN_COUPLING
118 opts.
addOpt< std::string >(
"ocean,m",
"ocean mesh filename (target)", &ocnFilename );
121 #ifdef ENABLE_ATMLND_COUPLING
122 opts.
addOpt< std::string >(
"land,l",
"land mesh filename (target)", &lndFilename );
125 opts.
addOpt< std::string >(
"physgrid,q",
"physics grid file", &atmPhysMesh );
127 opts.
addOpt<
int >(
"startAtm,a",
"start task for atmosphere layout", &
startG1 );
128 opts.
addOpt<
int >(
"endAtm,b",
"end task for atmosphere layout", &
endG1 );
129 #ifdef ENABLE_ATMOCN_COUPLING
130 opts.
addOpt<
int >(
"startOcn,c",
"start task for ocean layout", &
startG2 );
131 opts.
addOpt<
int >(
"endOcn,d",
"end task for ocean layout", &
endG2 );
134 #ifdef ENABLE_ATMLND_COUPLING
135 opts.
addOpt<
int >(
"startLnd,e",
"start task for land layout", &startG3 );
136 opts.
addOpt<
int >(
"endLnd,f",
"end task for land layout", &endG3 );
139 opts.
addOpt<
int >(
"startCoupler,g",
"start task for coupler layout", &startG4 );
140 opts.
addOpt<
int >(
"endCoupler,j",
"end task for coupler layout", &endG4 );
142 opts.
addOpt<
int >(
"partitioning,p",
"partitioning option for migration", &repartitioner_scheme );
151 #ifdef ENABLE_ATMOCN_COUPLING
152 "\n ocn file: " << ocnFilename <<
"\n on tasks : " <<
startG2 <<
":" <<
endG2 <<
154 #ifdef ENABLE_ATMLND_COUPLING
155 "\n land file: " << lndFilename <<
"\n on tasks : " << startG3 <<
":" << endG3 <<
158 "\n atm phys file: " << atmPhysMesh <<
"\n on tasks : " <<
startG1 <<
":" <<
endG1 <<
160 "\n partitioning (0 trivial, 1 graph, 2 geometry) " << repartitioner_scheme <<
"\n ";
166 MPI_Group atmPEGroup;
169 CHECKIERR(
ierr,
"Cannot create atm MPI group and communicator " )
171 #ifdef ENABLE_ATMOCN_COUPLING
172 MPI_Group ocnPEGroup;
175 CHECKIERR(
ierr,
"Cannot create ocn MPI group and communicator " )
178 #ifdef ENABLE_ATMLND_COUPLING
179 MPI_Group lndPEGroup;
182 CHECKIERR(
ierr,
"Cannot create lnd MPI group and communicator " )
186 MPI_Group couPEGroup;
189 CHECKIERR(
ierr,
"Cannot create cpl MPI group and communicator " )
195 MPI_Group joinAtmCouGroup;
200 #ifdef ENABLE_ATMOCN_COUPLING
202 MPI_Group joinOcnCouGroup;
205 CHECKIERR(
ierr,
"Cannot create joint ocn cou communicator" )
208 #ifdef ENABLE_ATMLND_COUPLING
210 MPI_Group joinLndCouGroup;
213 CHECKIERR(
ierr,
"Cannot create joint ocn cou communicator" )
219 int cmpAtmAppID = -1;
221 int cplAtmAppID = -1;
224 #ifdef ENABLE_ATMOCN_COUPLING
225 int cmpOcnAppID = -1;
227 int cplOcnAppID = -1, cplAtmOcnAppID = -1;
232 #ifdef ENABLE_ATMLND_COUPLING
233 int cmpLndAppID = -1;
235 int cplLndAppID = -1;
240 int cmpPhysAtmID = -1;
243 if( couComm != MPI_COMM_NULL )
245 MPI_Comm_rank( couComm, &rankInCouComm );
250 #ifdef ENABLE_ATMOCN_COUPLING
256 #ifdef ENABLE_ATMLND_COUPLING
263 if( atmCouComm != MPI_COMM_NULL )
269 #ifdef ENABLE_ATMOCN_COUPLING
270 if( ocnComm != MPI_COMM_NULL )
272 MPI_Comm_rank( ocnComm, &rankInOcnComm );
286 #ifdef ENABLE_ATMOCN_COUPLING
290 &couPEGroup, &ocnCouComm, ocnFilename,
readopts,
nghlay, repartitioner_scheme );
293 if( couComm != MPI_COMM_NULL )
295 char outputFileTgt3[] =
"recvOcn2.h5m";
306 if( atmComm != MPI_COMM_NULL )
315 int nverts[3], nelem[3];
318 printf( "Phys Atm Component Mesh: %d vertices and %d elements\n", nverts[0], nelem[0] );
323 #ifdef ENABLE_ATMLND_COUPLING
325 if( lndComm != MPI_COMM_NULL )
331 &couPEGroup, &lndCouComm, lndFilename, readoptsPhysAtm,
nghlay,
332 repartitioner_scheme );
333 if( couComm != MPI_COMM_NULL )
335 char outputFileLnd[] =
"recvLnd2.h5m";
345 #ifdef ENABLE_ATMOCN_COUPLING
346 if( couComm != MPI_COMM_NULL )
350 CHECKIERR(
ierr,
"Cannot register ocn_atm intx over coupler pes " )
354 const char* weights_identifiers[2] = {
"scalar",
"scalar-pc" };
355 int disc_orders[3] = { 4, 1, 1 };
356 const char* disc_methods[3] = {
"cgll",
"fv",
"pcloud" };
357 const char* dof_tag_names[3] = {
"GLOBAL_DOFS",
"GLOBAL_ID",
"GLOBAL_ID" };
358 #ifdef ENABLE_ATMOCN_COUPLING
359 if( couComm != MPI_COMM_NULL )
361 ierr = iMOAB_ComputeMeshIntersectionOnSphere( cplAtmPID, cplOcnPID, cplAtmOcnPID );
368 if( atmCouComm != MPI_COMM_NULL )
378 ierr = iMOAB_CoverageGraph( &atmCouComm, cmpAtmPID, cplAtmPID, cplAtmOcnPID, &
cmpatm, &cplatm,
380 CHECKIERR(
ierr,
"cannot recompute direct coverage graph for ocean" )
384 if( atmCouComm != MPI_COMM_NULL )
388 ierr = iMOAB_ComputeCommGraph( cmpPhAtmPID, cplAtmOcnPID, &atmCouComm, &atmPEGroup, &couPEGroup, &
typeA, &
typeB,
393 #ifdef ENABLE_ATMLND_COUPLING
397 if( atmCouComm != MPI_COMM_NULL )
401 ierr = iMOAB_ComputeCommGraph( cmpPhAtmPID, cplLndPID, &atmCouComm, &atmPEGroup, &couPEGroup, &
typeA, &
typeB,
407 int fMonotoneTypeID = 0, fVolumetric = 0, fValidate = 1, fNoConserve = 0, fNoBubble = 1, fInverseDistanceMap = 0;
409 #ifdef ENABLE_ATMOCN_COUPLING
411 if( couComm != MPI_COMM_NULL )
413 char serialWriteOptions[] =
"";
414 std::stringstream outf;
415 outf <<
"intxAtmOcn_" << rankInCouComm <<
".h5m";
416 std::string intxfile = outf.str();
422 if( couComm != MPI_COMM_NULL )
424 PUSH_TIMER(
"Compute the projection weights with TempestRemap" )
425 ierr = iMOAB_ComputeScalarProjectionWeights( cplAtmOcnPID, weights_identifiers[0], disc_methods[0],
426 &disc_orders[0], disc_methods[1], &disc_orders[1],
nullptr,
427 &fNoBubble, &fMonotoneTypeID, &fVolumetric, &fInverseDistanceMap,
428 &fNoConserve, &fValidate, dof_tag_names[0], dof_tag_names[1] );
429 CHECKIERR(
ierr, "cannot compute scalar projection weights" )
439 int atmCompNDoFs = disc_orders[0] * disc_orders[0], ocnCompNDoFs = 1 ;
441 const char* bottomFields =
442 "Sa_z:Sa_topo:Sa_u:Sa_v:Sa_tbot:Sa_ptem:Sa_shum:Sa_pbot:Sa_dens:Sa_uovern:Sa_pslv:Sa_co2prog:Sa_co2diag:Faxa_"
443 "rainc:Faxa_rainl:Faxa_snowc:Faxa_snowl:Faxa_lwdn:Faxa_swndr:Faxa_swvdr:Faxa_swndf:Faxa_swvdf:Faxa_swnet:Faxa_"
444 "bcphidry:Faxa_bcphodry:Faxa_bcphiwet:Faxa_ocphidry:Faxa_ocphodry:Faxa_ocphiwet:Faxa_dstwet1:Faxa_dstwet2:Faxa_"
445 "dstwet3:Faxa_dstwet4:Faxa_dstdry1:Faxa_dstdry2:Faxa_dstdry3:Faxa_dstdry4";
446 const char* bottomFieldsExt =
447 "Sa_z_ext:Sa_topo_ext:Sa_u_ext:Sa_v_ext:Sa_tbot_ext:Sa_ptem_ext:Sa_shum_ext:Sa_pbot_ext:Sa_dens_ext:Sa_uovern_"
448 "ext:Sa_pslv_ext:Sa_co2prog_ext:Sa_co2diag_ext:Faxa_rainc_ext:Faxa_rainl_ext:Faxa_snowc_ext:Faxa_snowl_ext:"
449 "Faxa_lwdn_ext:Faxa_swndr_ext:Faxa_swvdr_ext:Faxa_swndf_ext:Faxa_swvdf_ext:Faxa_swnet_ext:Faxa_bcphidry_ext:"
450 "Faxa_bcphodry_ext:Faxa_bcphiwet_ext:Faxa_ocphidry_ext:Faxa_ocphodry_ext:Faxa_ocphiwet_ext:Faxa_dstwet1_ext:"
451 "Faxa_dstwet2_ext:Faxa_dstwet3_ext:Faxa_dstwet4_ext:Faxa_dstdry1_ext:Faxa_dstdry2_ext:Faxa_dstdry3_ext:Faxa_"
454 if( couComm != MPI_COMM_NULL )
457 CHECKIERR(
ierr,
"failed to define the field tags T16_ph:u16_ph:v16_ph " );
458 #ifdef ENABLE_ATMOCN_COUPLING
461 CHECKIERR(
ierr,
"failed to define the field tags T_proj:u_proj:v_proj" );
464 #ifdef ENABLE_ATMLND_COUPLING
469 CHECKIERR(
ierr,
"failed to define the bottomFields tag on coupler land" );
480 if( cplAtmAppID >= 0 )
482 int nverts[3], nelem[3], nblocks[3], nsbc[3], ndbc[3];
492 int numAllElem = nelem[2];
493 std::vector< double > vals;
494 int storLeng = atmCompNDoFs * numAllElem * 37;
495 vals.resize( storLeng );
496 for(
int k = 0; k < storLeng; k++ )
505 #ifdef ENABLE_ATMOCN_COUPLING
506 PUSH_TIMER(
"Send/receive data from atm component to coupler in ocn context" )
507 if( atmComm != MPI_COMM_NULL )
511 ierr = iMOAB_SendElementTag( cmpPhAtmPID, bottomFields, &atmCouComm, &atmocnid );
516 iMOAB_DumpCommGraph( cmpPhAtmPID, &
context, &is_sender,
"PhysAtmA2OS", );
519 if( couComm != MPI_COMM_NULL )
522 ierr = iMOAB_ReceiveElementTag( cplAtmOcnPID, bottomFieldsExt, &atmCouComm, &
cmpatm );
527 iMOAB_DumpCommGraph( cplAtmOcnPID, &
context, &is_sender,
"PhysAtmA2OR", );
533 if( atmComm != MPI_COMM_NULL )
535 ierr = iMOAB_FreeSenderBuffers( cmpPhAtmPID, &atmocnid );
536 CHECKIERR(
ierr,
"cannot free buffers used to resend atm tag towards the coverage mesh" )
548 if( couComm != MPI_COMM_NULL )
552 PUSH_TIMER(
"Apply Scalar projection weights" )
554 iMOAB_ApplyScalarProjectionWeights( cplAtmOcnPID, weights_identifiers[0], bottomFieldsExt, bottomFields );
555 CHECKIERR(
ierr, "failed to compute projection weight application" );
558 char outputFileTgt[] = "fOcnOnCpl7.h5m";
562 if( ocnComm != MPI_COMM_NULL )
566 CHECKIERR(
ierr,
"failed to define the field tags for receiving back the atm tags on ocn pes" );
572 if( couComm != MPI_COMM_NULL )
575 ierr = iMOAB_SendElementTag( cplOcnPID, bottomFields, &ocnCouComm, &context_id );
576 CHECKIERR(
ierr,
"cannot send tag values back to ocean pes" )
580 if( ocnComm != MPI_COMM_NULL )
583 ierr = iMOAB_ReceiveElementTag( cmpOcnPID, bottomFields, &ocnCouComm, &context_id );
584 CHECKIERR(
ierr,
"cannot receive tag values from ocean mesh on coupler pes" )
588 if( couComm != MPI_COMM_NULL )
591 ierr = iMOAB_FreeSenderBuffers( cplOcnPID, &context_id );
593 if( ocnComm != MPI_COMM_NULL )
595 char outputFileOcn[] =
"OcnWithProj2.h5m";
601 #ifdef ENABLE_ATMLND_COUPLING
609 PUSH_TIMER(
"Send/receive data from phys comp atm to coupler land, using computed graph" )
610 if( atmComm != MPI_COMM_NULL )
615 ierr = iMOAB_SendElementTag( cmpPhAtmPID, bottomFields, &atmCouComm, &cpllnd );
616 CHECKIERR(
ierr,
"cannot send tag values towards cpl on land" )
618 if( couComm != MPI_COMM_NULL )
621 ierr = iMOAB_ReceiveElementTag( cplLndPID, bottomFields, &atmCouComm, &
cmpatm );
622 CHECKIERR(
ierr,
"cannot receive tag values on land on coupler" )
627 if( atmComm != MPI_COMM_NULL )
629 ierr = iMOAB_FreeSenderBuffers( cmpPhAtmPID, &cpllnd );
630 CHECKIERR(
ierr,
"cannot free buffers used to send atm tag towards the land on coupler" )
634 if( couComm != MPI_COMM_NULL )
636 char outputFileTgtLnd[] =
"fLndOnCpl.h5m";
649 if( lndComm != MPI_COMM_NULL )
653 CHECKIERR(
ierr,
"failed to define the field tag for receiving back the tag T_proj:u_proj:v_proj on lnd pes" );
655 if( couComm != MPI_COMM_NULL )
658 ierr = iMOAB_SendElementTag( cplLndPID, bottomFields, &lndCouComm, &context_id );
662 if( lndComm != MPI_COMM_NULL )
665 ierr = iMOAB_ReceiveElementTag( cmpLndPID, bottomFields, &lndCouComm, &context_id );
666 CHECKIERR(
ierr,
"cannot receive tag values from land mesh on coupler pes" )
670 if( couComm != MPI_COMM_NULL )
673 ierr = iMOAB_FreeSenderBuffers( cplLndPID, &context_id );
675 if( lndComm != MPI_COMM_NULL )
677 char outputFileLnd[] =
"LndWithProj2.h5m";
683 #ifdef ENABLE_ATMOCN_COUPLING
684 if( couComm != MPI_COMM_NULL )
691 #ifdef ENABLE_ATMLND_COUPLING
692 if( lndComm != MPI_COMM_NULL )
698 if( atmComm != MPI_COMM_NULL )
703 #ifdef ENABLE_ATMOCN_COUPLING
704 if( ocnComm != MPI_COMM_NULL )
711 if( atmComm != MPI_COMM_NULL )
717 #ifdef ENABLE_ATMLND_COUPLING
718 if( couComm != MPI_COMM_NULL )
725 #ifdef ENABLE_ATMOCN_COUPLING
726 if( couComm != MPI_COMM_NULL )
733 if( couComm != MPI_COMM_NULL )
744 if( MPI_COMM_NULL != atmCouComm ) MPI_Comm_free( &atmCouComm );
745 MPI_Group_free( &joinAtmCouGroup );
746 if( MPI_COMM_NULL != atmComm ) MPI_Comm_free( &atmComm );
748 #ifdef ENABLE_ATMOCN_COUPLING
749 if( MPI_COMM_NULL != ocnComm ) MPI_Comm_free( &ocnComm );
751 if( MPI_COMM_NULL != ocnCouComm ) MPI_Comm_free( &ocnCouComm );
752 MPI_Group_free( &joinOcnCouGroup );
755 #ifdef ENABLE_ATMLND_COUPLING
756 if( MPI_COMM_NULL != lndComm ) MPI_Comm_free( &lndComm );
758 if( MPI_COMM_NULL != lndCouComm ) MPI_Comm_free( &lndCouComm );
759 MPI_Group_free( &joinLndCouGroup );
762 if( MPI_COMM_NULL != couComm ) MPI_Comm_free( &couComm );
764 MPI_Group_free( &atmPEGroup );
765 #ifdef ENABLE_ATMOCN_COUPLING
766 MPI_Group_free( &ocnPEGroup );
768 #ifdef ENABLE_ATMLND_COUPLING
769 MPI_Group_free( &lndPEGroup );
771 MPI_Group_free( &couPEGroup );
772 MPI_Group_free( &
jgroup );