MOAB: Mesh Oriented datABase  (version 5.5.0)
Functions
imoab_scrip.cpp File Reference
#include "moab/Core.hpp"
#include "moab_mpi.h"
#include "moab/ParallelComm.hpp"
#include "MBParallelConventions.h"
#include "moab/iMOAB.h"
#include "TestUtil.hpp"
#include "moab/CpuTimer.hpp"
#include "moab/ProgOptions.hpp"
#include <iostream>
#include <sstream>
#include "imoab_coupler_utils.hpp"
+ Include dependency graph for imoab_scrip.cpp:

Go to the source code of this file.

Functions

int main (int argc, char *argv[])
 

Function Documentation

◆ main()

int main ( int  argc,
char *  argv[] 
)

Definition at line 24 of file imoab_scrip.cpp.

25 {
26  int ierr;
27  int rankInGlobalComm, numProcesses;
28  MPI_Group jgroup;
29  std::string readopts2( "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN" );
30  std::string readopts( "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS" );
31  std::string readoptsLnd( "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION" );
32  std::string filename = TestDir + "unittest/SCRIPgrid_2x2_nomask_c210211.nc";
33  std::string atmFilename = TestDir + "unittest/wholeATM_T.h5m";
34  std::string rofInp = TestDir + "unittest/wholeRof_06.h5m";
35  std::string seq_flds_r2x_fields(
36  "Forr_rofl:Forr_rofi:Firr_rofi:Flrr_flood:Flrr_volr:Flrr_volrmch:Flrr_supply:Flrr_deficit" );
37  int cmpAtm = 5, cmpRof = 21, cplRof = 22;
38  int cplatm = 6; // component ids are unique over all pes, and established in advance;
39  int nghlay = 0; // no ghost layers
40 
41  MPI_Init( &argc, &argv );
42  MPI_Comm_rank( MPI_COMM_WORLD, &rankInGlobalComm );
43  MPI_Comm_size( MPI_COMM_WORLD, &numProcesses );
44  MPI_Comm_group( MPI_COMM_WORLD, &jgroup ); // all processes in jgroup
45 
46  int startG1 = 0, startG2 = 0, startG4 = 0;
47  int endG1, endG2, endG4;
48  endG1 = endG2 = endG4 = numProcesses - 1;
49 
50  ProgOptions opts;
51  opts.addOpt< std::string >( "atmosphere,t", "atm mesh filename ", &atmFilename );
52 
53  opts.addOpt< std::string >( "mosart,m", " mosart with data", &rofInp );
54 
55  opts.addOpt< std::string >( "scrip,s", "scrip mesh file", &filename );
56 
57  opts.addOpt< int >( "startAtm,a", "start task for atmosphere layout", &startG1 );
58  opts.addOpt< int >( "endAtm,b", "end task for atmosphere layout", &endG1 );
59 
60  opts.addOpt< int >( "startOcn,c", "start task for mosart layout", &startG2 );
61  opts.addOpt< int >( "endOcn,d", "end task for mosart layout", &endG2 );
62 
63  opts.addOpt< int >( "startCoupler,g", "start task for coupler layout", &startG4 );
64  opts.addOpt< int >( "endCoupler,j", "end task for coupler layout", &endG4 );
65 
66  opts.parseCommandLine( argc, argv );
67 
68  if( !rankInGlobalComm )
69  {
70  std::cout << " atm file: " << atmFilename << "\n on tasks : " << startG1 << ":" << endG1
71  << "\n mosart input file file: " << rofInp << "\n on tasks : " << startG2 << ":" << endG2
72  << "\n scrip file on coupler: " << filename << "\n coupler on tasks : " << startG4 << ":" << endG4
73  << "\n";
74  }
75 
76  // load files on 2 different communicators, groups
77  // coupler will be on group 4
78  MPI_Group atmPEGroup;
79  MPI_Comm atmComm;
80  ierr = create_group_and_comm( startG1, endG1, jgroup, &atmPEGroup, &atmComm );
81  CHECKIERR( ierr, "Cannot create atm MPI group and communicator " )
82 
83  MPI_Group rofPEGroup;
84  MPI_Comm rofComm;
85  ierr = create_group_and_comm( startG2, endG2, jgroup, &rofPEGroup, &rofComm );
86  CHECKIERR( ierr, "Cannot create rof MPI group and communicator " )
87 
88  // we will always have a coupler
89  MPI_Group couPEGroup;
90  MPI_Comm couComm;
91  ierr = create_group_and_comm( startG4, endG4, jgroup, &couPEGroup, &couComm );
92  CHECKIERR( ierr, "Cannot create cpl MPI group and communicator " )
93 
94  // atm_coupler
95  MPI_Group joinAtmCouGroup;
96  MPI_Comm atmCouComm;
97  ierr = create_joint_comm_group( atmPEGroup, couPEGroup, &joinAtmCouGroup, &atmCouComm );
98  CHECKIERR( ierr, "Cannot create joint atm cou communicator" )
99 
100  // rof_coupler
101  MPI_Group joinRofCouGroup;
102  MPI_Comm rofCouComm;
103  ierr = create_joint_comm_group( rofPEGroup, couPEGroup, &joinRofCouGroup, &rofCouComm );
104  CHECKIERR( ierr, "Cannot create joint rof cou communicator" )
105 
106  ierr = iMOAB_Initialize( argc, argv ); // not really needed anything from argc, argv, yet; maybe we should
107  CHECKIERR( ierr, "Cannot initialize iMOAB" )
108 
109  int cmpRofID = -1;
110  iMOAB_AppID rofPID = &cmpRofID;
111  if( rofComm != MPI_COMM_NULL )
112  {
113  ierr = iMOAB_RegisterApplication( "ROF", &rofComm, &cmpRof, rofPID );
114  CHECKIERR( ierr, "Cannot register Rof App" )
115  }
116 
117  int cmpAtmAppID = -1;
118  iMOAB_AppID cmpAtmPID = &cmpAtmAppID;
119  if( atmComm != MPI_COMM_NULL )
120  {
121  ierr = iMOAB_RegisterApplication( "ATM", &atmComm, &cmpAtm, cmpAtmPID );
122  CHECKIERR( ierr, "Cannot register Atm App" )
123  }
124  int cplAtmAppID = -1;
125  iMOAB_AppID cplAtmPID = &cplAtmAppID;
126 
127  int cplRofAppID = -1;
128  iMOAB_AppID cplRofPID = &cplRofAppID;
129 
130  int rankInCouComm = -1;
131  if( couComm != MPI_COMM_NULL )
132  {
133  MPI_Comm_rank( couComm, &rankInCouComm );
134  // Register all the applications on the coupler PEs
135  ierr = iMOAB_RegisterApplication( "ATMX", &couComm, &cplatm,
136  cplAtmPID ); // atm on coupler pes
137  CHECKIERR( ierr, "Cannot register ATM over coupler PEs" )
138 
139  ierr = iMOAB_RegisterApplication( "ROFX", &couComm, &cplRof,
140  cplRofPID ); // ocn on coupler pes
141  CHECKIERR( ierr, "Cannot register ROFX over coupler PEs" )
142  }
143 
144  int repartitioner_scheme = 0;
145 #ifdef MOAB_HAVE_ZOLTAN
146  repartitioner_scheme = 2; // zoltan is used
147 #endif
148  if( atmCouComm != MPI_COMM_NULL )
149  {
150  ierr = setup_component_coupler_meshes( cmpAtmPID, cmpAtm, cplAtmPID, cplatm, &atmComm, &atmPEGroup, &couComm,
151  &couPEGroup, &atmCouComm, atmFilename, readopts, nghlay,
152  repartitioner_scheme );
153  CHECKIERR( ierr, "Cannot load and migrate atm mesh " )
154  }
155  int tagtype = 1, numco = 1, tagIndex = 0;
156  if( cmpRofID >= 0 )
157  {
158  // load rof mesh with data on it
159  ierr = iMOAB_LoadMesh( rofPID, rofInp.c_str(), readoptsLnd.c_str(), &nghlay );
160  CHECKIERR( ierr, "Cannot load mosart data mesh" )
161 
162  ierr = iMOAB_DefineTagStorage( rofPID, seq_flds_r2x_fields.c_str(), &tagtype, &numco, &tagIndex );
163  CHECKIERR( ierr, "failed to define the fields on mosart point cloud" )
164  }
165  // load rof scrip file on coupler only
166  if( couComm != MPI_COMM_NULL )
167  {
168  ierr = iMOAB_LoadMesh( cplRofPID, filename.c_str(), readopts2.c_str(), &nghlay );
169  CHECKIERR( ierr, "Cannot load scrip mesh on coupler" )
170  // define tags on receiving end
171  ierr = iMOAB_DefineTagStorage( cplRofPID, seq_flds_r2x_fields.c_str(), &tagtype, &numco, &tagIndex );
172  CHECKIERR( ierr, "failed to define the fields on mosart coupler mesh " )
173  // test what we read from scrip file
174  char outputFileTgt[] = "readCplRof.h5m";
175  char fileWriteOptions[] = "PARALLEL=WRITE_PART";
176  ierr = iMOAB_WriteMesh( cplRofPID, outputFileTgt, fileWriteOptions );
177  CHECKIERR( ierr, "cannot write Rof mesh on coupler" )
178  }
179  // compute comm graph between coupler and wholeRof
180  if( MPI_COMM_NULL != rofCouComm )
181  {
182  // compute the comm graph between point cloud rof and coupler version of rof (full mesh)
183  // we are now on joint pes, compute comm graph between rof and coupler model
184  int typeA = 2; // point cloud on component PEs
185  int typeB = 3; // full mesh on coupler pes, we just read it
186  ierr = iMOAB_ComputeCommGraph( rofPID, cplRofPID, &rofCouComm, &rofPEGroup, &couPEGroup, &typeA, &typeB,
187  &cmpRof, &cplRof );
188  CHECKIERR( ierr, "cannot compute comm graph for mosart " )
189  }
190 
191  // now send / receive some tags
192  if( cmpRofID >= 0 )
193  { // send
194  // basically, use the initial partitioning
195  ierr = iMOAB_SendElementTag( rofPID, seq_flds_r2x_fields.c_str(), &rofCouComm, &cplRof );
196  CHECKIERR( ierr, "cannot send tags " )
197  }
198 
199  if( cplRofAppID >= 0 )
200  { // we are on receiving end
201  ierr = iMOAB_ReceiveElementTag( cplRofPID, seq_flds_r2x_fields.c_str(), &rofCouComm, &cmpRof );
202  CHECKIERR( ierr, "cannot receive tags " )
203  char outputFileTgt[] = "afterSend.h5m";
204  char fileWriteOptions[] = "PARALLEL=WRITE_PART";
205  ierr = iMOAB_WriteMesh( cplRofPID, outputFileTgt, fileWriteOptions );
206  CHECKIERR( ierr, "cannot write Rof mesh with data on coupler" )
207  }
208 
209  if( cmpRofID >= 0 )
210  { // send
211  // basically, use the initial partitioning
212  ierr = iMOAB_FreeSenderBuffers( rofPID, &cplRof );
213  CHECKIERR( ierr, "cannot free buffers " )
214  }
215  MPI_Finalize();
216 
217  return 0;
218 }

References ProgOptions::addOpt(), atmFilename, CHECKIERR, create_group_and_comm(), create_joint_comm_group(), endG1, endG2, filename, fileWriteOptions(), ierr, iMOAB_AppID, iMOAB_DefineTagStorage(), iMOAB_Initialize(), iMOAB_LoadMesh(), iMOAB_RegisterApplication(), iMOAB_WriteMesh(), jgroup, MPI_COMM_WORLD, nghlay, numProcesses, ProgOptions::parseCommandLine(), rankInGlobalComm, readopts(), setup_component_coupler_meshes(), startG1, startG2, typeA, and typeB.