MOAB: Mesh Oriented datABase  (version 5.5.0)
write_nc.cpp
Go to the documentation of this file.
1 #include "TestUtil.hpp"
2 #include "moab/Core.hpp"
3 
4 using namespace moab;
5 
6 std::string example_eul = TestDir + "unittest/io/eul3x48x96.t.3.nc";
7 std::string example_eul_t0 = TestDir + "unittest/io/eul3x48x96.t0.nc";
8 std::string example_eul_t1 = TestDir + "unittest/io/eul3x48x96.t1.nc";
9 std::string example_eul_t2 = TestDir + "unittest/io/eul3x48x96.t2.nc";
10 std::string example_fv = TestDir + "unittest/io/fv3x46x72.t.3.nc";
11 std::string example_homme = TestDir + "unittest/io/homme3x3458.t.3.nc";
12 std::string example_mpas = TestDir + "unittest/io/mpasx1.642.t.2.nc";
13 std::string example_gcrm = TestDir + "unittest/io/gcrm_r3.nc";
14 
15 #ifdef MOAB_HAVE_MPI
16 #include "moab_mpi.h"
17 #include "moab/ParallelComm.hpp"
18 #endif
19 
20 #ifdef MOAB_HAVE_PNETCDF
21 #include "pnetcdf.h"
22 #define NCFUNC( func ) ncmpi_##func
23 #define NCDF_SIZE MPI_Offset
24 #else
25 #include "netcdf.h"
26 #define NCFUNC( func ) nc_##func
27 #define NCDF_SIZE size_t
28 #endif
29 
30 // CAM-EUL
31 void test_eul_read_write_T();
32 void test_eul_check_T();
33 
34 // CAM-FV
35 void test_fv_read_write_T();
36 void test_fv_check_T();
37 
38 // CAM-SE (HOMME)
39 void test_homme_read_write_T();
40 void test_homme_check_T();
41 
42 // MPAS
43 void test_mpas_read_write_vars();
44 void test_mpas_check_vars();
45 
46 // GCRM
47 void test_gcrm_read_write_vars();
48 void test_gcrm_check_vars();
49 
50 // Test timestep option
51 void test_eul_read_write_timestep();
52 void test_eul_check_timestep();
53 
54 // Test append option
55 void test_eul_read_write_append();
56 void test_eul_check_append();
57 
58 // Test writing variables with timesteps spread across files
59 void test_eul_read_write_across_files();
60 void test_eul_check_across_files();
61 
62 #ifdef MOAB_HAVE_MPI
63 // Test mesh with ghosted entities
64 void test_eul_read_write_ghosting();
65 void test_eul_check_ghosting();
66 #endif
67 
68 void get_eul_read_options( std::string& opts );
69 void get_fv_read_options( std::string& opts );
70 void get_homme_read_options( std::string& opts );
71 void get_mpas_read_options( std::string& opts );
72 
73 const double eps = 1e-10;
74 const int levels = 3;
75 const int mpas_levels = 1;
76 
77 int main( int argc, char* argv[] )
78 {
79  int result = 0;
80 
81 #ifdef MOAB_HAVE_MPI
82  int fail = MPI_Init( &argc, &argv );
83  if( fail ) return 1;
84 #else
85  argv[0] = argv[argc - argc]; // To remove the warnings in serial mode about unused variables
86 #endif
87 
88  result += RUN_TEST( test_eul_read_write_T );
89  result += RUN_TEST( test_eul_check_T );
90 
91  result += RUN_TEST( test_fv_read_write_T );
92  result += RUN_TEST( test_fv_check_T );
93 
94  result += RUN_TEST( test_homme_read_write_T );
95  result += RUN_TEST( test_homme_check_T );
96 
97  result += RUN_TEST( test_mpas_read_write_vars );
98  result += RUN_TEST( test_mpas_check_vars );
99 
100  result += RUN_TEST( test_gcrm_read_write_vars );
101  result += RUN_TEST( test_gcrm_check_vars );
102 
103  result += RUN_TEST( test_eul_read_write_timestep );
104  result += RUN_TEST( test_eul_check_timestep );
105 
106  result += RUN_TEST( test_eul_read_write_append );
107  result += RUN_TEST( test_eul_check_append );
108 
109  result += RUN_TEST( test_eul_read_write_across_files );
110  result += RUN_TEST( test_eul_check_across_files );
111 
112 #ifdef MOAB_HAVE_MPI
113  result += RUN_TEST( test_eul_read_write_ghosting );
114  result += RUN_TEST( test_eul_check_ghosting );
115 #endif
116 
117 #ifdef MOAB_HAVE_MPI
118  fail = MPI_Finalize();
119  if( fail ) return 1;
120 #endif
121 
122  return result;
123 }
124 
125 // We also read and write gw (test writing a set variable without timesteps)
126 void test_eul_read_write_T()
127 {
128  int procs = 1;
129 #ifdef MOAB_HAVE_MPI
130  MPI_Comm_size( MPI_COMM_WORLD, &procs );
131 #endif
132 
133 // We will not test NC writer in parallel without pnetcdf support
134 #ifndef MOAB_HAVE_PNETCDF
135  if( procs > 1 ) return;
136 #endif
137 
138  Core moab;
139  Interface& mb = moab;
140 
141  std::string read_opts;
142  get_eul_read_options( read_opts );
143 
144  EntityHandle set;
145  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
146 
147  // Read non-set variable T and set variable gw
148  read_opts += ";VARIABLE=T,gw;DEBUG_IO=0";
149  rval = mb.load_file( example_eul.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
150 
151  // Write variables T and gw
152  std::string write_opts = ";;VARIABLE=T,gw;DEBUG_IO=0";
153 #ifdef MOAB_HAVE_MPI
154  // Use parallel options
155  write_opts += ";PARALLEL=WRITE_PART";
156 #endif
157  if( procs > 1 )
158  rval = mb.write_file( "test_par_eul_T.nc", 0, write_opts.c_str(), &set, 1 );
159  else
160  rval = mb.write_file( "test_eul_T.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
161 }
162 
163 // Check non-set variable T
164 // Also check set variable gw
165 void test_eul_check_T()
166 {
167  int rank = 0;
168  int procs = 1;
169 #ifdef MOAB_HAVE_MPI
170  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
171  MPI_Comm_size( MPI_COMM_WORLD, &procs );
172 #endif
173 
174 // We will not test NC writer in parallel without pnetcdf support
175 #ifndef MOAB_HAVE_PNETCDF
176  if( procs > 1 ) return;
177 #endif
178 
179  if( 0 == rank )
180  {
181  int ncid;
182  int ncid_ref;
183  int success;
184 
185  std::string filename;
186  if( procs > 1 )
187  filename = "test_par_eul_T.nc";
188  else
189  filename = "test_eul_T.nc";
190 
191 #ifdef MOAB_HAVE_PNETCDF
192  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
193 #else
194  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
195 #endif
196  CHECK_EQUAL( 0, success );
197 
198 #ifdef MOAB_HAVE_PNETCDF
199  success = NCFUNC( open )( MPI_COMM_SELF, example_eul.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
200 #else
201  success = NCFUNC( open )( example_eul.c_str(), NC_NOWRITE, &ncid_ref );
202 #endif
203  CHECK_EQUAL( 0, success );
204 
205  int T_id;
206  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
207  CHECK_EQUAL( 0, success );
208 
209  int T_id_ref;
210  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
211  CHECK_EQUAL( 0, success );
212 
213  int gw_id;
214  success = NCFUNC( inq_varid )( ncid, "gw", &gw_id );
215  CHECK_EQUAL( 0, success );
216 
217  int gw_id_ref;
218  success = NCFUNC( inq_varid )( ncid_ref, "gw", &gw_id_ref );
219  CHECK_EQUAL( 0, success );
220 
221 #ifdef MOAB_HAVE_PNETCDF
222  // Enter independent I/O mode
223  success = NCFUNC( begin_indep_data )( ncid );
224  CHECK_EQUAL( 0, success );
225 #endif
226 
227 #ifdef MOAB_HAVE_PNETCDF
228  // Enter independent I/O mode
229  success = NCFUNC( begin_indep_data )( ncid_ref );
230  CHECK_EQUAL( 0, success );
231 #endif
232 
233  NCDF_SIZE start[] = { 0, 0, 0, 0 };
234  NCDF_SIZE count[] = { 3, levels, 48, 96 };
235  const int size = 3 * levels * 48 * 96;
236 
237  // Read variable T from output file
238  double T_vals[size];
239  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
240  CHECK_EQUAL( 0, success );
241 
242  // Read variable T from reference file
243  double T_vals_ref[size];
244  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
245  CHECK_EQUAL( 0, success );
246 
247  // Read variable gw (on lat) from output file
248  count[0] = 48;
249  double gw_vals[48];
250  success = NCFUNC( get_vara_double )( ncid, gw_id, start, count, gw_vals );
251  CHECK_EQUAL( 0, success );
252 
253  // Read variable gw (on lat) from reference file
254  double gw_vals_ref[48];
255  success = NCFUNC( get_vara_double )( ncid_ref, gw_id_ref, start, count, gw_vals_ref );
256  CHECK_EQUAL( 0, success );
257 
258 #ifdef MOAB_HAVE_PNETCDF
259  // End independent I/O mode
260  success = NCFUNC( end_indep_data )( ncid );
261  CHECK_EQUAL( 0, success );
262 #endif
263 
264 #ifdef MOAB_HAVE_PNETCDF
265  // End independent I/O mode
266  success = NCFUNC( end_indep_data )( ncid_ref );
267  CHECK_EQUAL( 0, success );
268 #endif
269 
270  success = NCFUNC( close )( ncid );
271  CHECK_EQUAL( 0, success );
272 
273  success = NCFUNC( close )( ncid_ref );
274  CHECK_EQUAL( 0, success );
275 
276  // Check T values
277  for( int i = 0; i < size; i++ )
278  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
279 
280  // Check gw values
281  for( int i = 0; i < 48; i++ )
282  CHECK_REAL_EQUAL( gw_vals_ref[i], gw_vals[i], eps );
283  }
284 }
285 
286 void test_fv_read_write_T()
287 {
288  int procs = 1;
289 #ifdef MOAB_HAVE_MPI
290  MPI_Comm_size( MPI_COMM_WORLD, &procs );
291 #endif
292 
293 // We will not test NC writer in parallel without pnetcdf support
294 #ifndef MOAB_HAVE_PNETCDF
295  if( procs > 1 ) return;
296 #endif
297 
298  Core moab;
299  Interface& mb = moab;
300 
301  std::string read_opts;
302  get_fv_read_options( read_opts );
303 
304  EntityHandle set;
305  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
306 
307  // Read non-set variable T
308  read_opts += ";VARIABLE=T;DEBUG_IO=0";
309  rval = mb.load_file( example_fv.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
310 
311  // Write variable T
312  std::string write_opts = ";;VARIABLE=T;DEBUG_IO=0";
313 #ifdef MOAB_HAVE_MPI
314  // Use parallel options
315  write_opts += ";PARALLEL=WRITE_PART";
316 #endif
317  if( procs > 1 )
318  rval = mb.write_file( "test_par_fv_T.nc", 0, write_opts.c_str(), &set, 1 );
319  else
320  rval = mb.write_file( "test_fv_T.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
321 }
322 
323 // Check non-set variable T
324 void test_fv_check_T()
325 {
326  int rank = 0;
327  int procs = 1;
328 #ifdef MOAB_HAVE_MPI
329  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
330  MPI_Comm_size( MPI_COMM_WORLD, &procs );
331 #endif
332 
333 // We will not test NC writer in parallel without pnetcdf support
334 #ifndef MOAB_HAVE_PNETCDF
335  if( procs > 1 ) return;
336 #endif
337 
338  if( 0 == rank )
339  {
340  int ncid;
341  int ncid_ref;
342  int success;
343 
344  std::string filename;
345  if( procs > 1 )
346  filename = "test_par_fv_T.nc";
347  else
348  filename = "test_fv_T.nc";
349 
350 #ifdef MOAB_HAVE_PNETCDF
351  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
352 #else
353  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
354 #endif
355  CHECK_EQUAL( 0, success );
356 
357 #ifdef MOAB_HAVE_PNETCDF
358  success = NCFUNC( open )( MPI_COMM_SELF, example_fv.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
359 #else
360  success = NCFUNC( open )( example_fv.c_str(), NC_NOWRITE, &ncid_ref );
361 #endif
362  CHECK_EQUAL( 0, success );
363 
364  int T_id;
365  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
366  CHECK_EQUAL( 0, success );
367 
368  int T_id_ref;
369  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
370  CHECK_EQUAL( 0, success );
371 
372 #ifdef MOAB_HAVE_PNETCDF
373  // Enter independent I/O mode
374  success = NCFUNC( begin_indep_data )( ncid );
375  CHECK_EQUAL( 0, success );
376 #endif
377 
378 #ifdef MOAB_HAVE_PNETCDF
379  // Enter independent I/O mode
380  success = NCFUNC( begin_indep_data )( ncid_ref );
381  CHECK_EQUAL( 0, success );
382 #endif
383 
384  NCDF_SIZE start[] = { 0, 0, 0, 0 };
385  NCDF_SIZE count[] = { 3, levels, 46, 72 };
386  const int size = 3 * levels * 46 * 72;
387 
388  // Read variable T from output file
389  double T_vals[size];
390  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
391  CHECK_EQUAL( 0, success );
392 
393  // Read variable T from reference file
394  double T_vals_ref[size];
395  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
396  CHECK_EQUAL( 0, success );
397 
398 #ifdef MOAB_HAVE_PNETCDF
399  // End independent I/O mode
400  success = NCFUNC( end_indep_data )( ncid );
401  CHECK_EQUAL( 0, success );
402 #endif
403 
404 #ifdef MOAB_HAVE_PNETCDF
405  // End independent I/O mode
406  success = NCFUNC( end_indep_data )( ncid_ref );
407  CHECK_EQUAL( 0, success );
408 #endif
409 
410  success = NCFUNC( close )( ncid );
411  CHECK_EQUAL( 0, success );
412 
413  success = NCFUNC( close )( ncid_ref );
414  CHECK_EQUAL( 0, success );
415 
416  // Check T values
417  for( int i = 0; i < size; i++ )
418  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
419  }
420 }
421 
422 // We also read and write lat (test writing a set variable without timesteps)
423 void test_homme_read_write_T()
424 {
425  int procs = 1;
426 #ifdef MOAB_HAVE_MPI
427  MPI_Comm_size( MPI_COMM_WORLD, &procs );
428 #endif
429 
430 // We will not test NC writer in parallel without pnetcdf support
431 #ifndef MOAB_HAVE_PNETCDF
432  if( procs > 1 ) return;
433 #endif
434 
435  Core moab;
436  Interface& mb = moab;
437 
438  std::string read_opts;
439  get_homme_read_options( read_opts );
440 
441  EntityHandle set;
442  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
443 
444  // Read non-set variable T and set variable lat
445  read_opts += ";VARIABLE=T,lat;DEBUG_IO=0";
446  if( procs > 1 )
447  {
448  // Rotate trivial partition, otherwise localGidVertsOwned.psize() is always 1
449  read_opts += ";PARALLEL_RESOLVE_SHARED_ENTS;TRIVIAL_PARTITION_SHIFT=1";
450  }
451  rval = mb.load_file( example_homme.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
452 
453  // Write variables T and lat
454  std::string write_opts = ";;VARIABLE=T,lat;DEBUG_IO=0";
455 #ifdef MOAB_HAVE_MPI
456  // Use parallel options
457  write_opts += ";PARALLEL=WRITE_PART";
458 #endif
459  if( procs > 1 )
460  rval = mb.write_file( "test_par_homme_T.nc", 0, write_opts.c_str(), &set, 1 );
461  else
462  rval = mb.write_file( "test_homme_T.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
463 }
464 
465 // Check non-set variable T
466 // Also check set variable lat
467 void test_homme_check_T()
468 {
469  int rank = 0;
470  int procs = 1;
471 #ifdef MOAB_HAVE_MPI
472  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
473  MPI_Comm_size( MPI_COMM_WORLD, &procs );
474 #endif
475 
476 // We will not test NC writer in parallel without pnetcdf support
477 #ifndef MOAB_HAVE_PNETCDF
478  if( procs > 1 ) return;
479 #endif
480 
481  if( 0 == rank )
482  {
483  int ncid;
484  int ncid_ref;
485  int success;
486 
487  std::string filename;
488  if( procs > 1 )
489  filename = "test_par_homme_T.nc";
490  else
491  filename = "test_homme_T.nc";
492 
493 #ifdef MOAB_HAVE_PNETCDF
494  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
495 #else
496  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
497 #endif
498  CHECK_EQUAL( 0, success );
499 
500 #ifdef MOAB_HAVE_PNETCDF
501  success = NCFUNC( open )( MPI_COMM_SELF, example_homme.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
502 #else
503  success = NCFUNC( open )( example_homme.c_str(), NC_NOWRITE, &ncid_ref );
504 #endif
505  CHECK_EQUAL( 0, success );
506 
507  int T_id;
508  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
509  CHECK_EQUAL( 0, success );
510 
511  int T_id_ref;
512  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
513  CHECK_EQUAL( 0, success );
514 
515  int lat_id;
516  success = NCFUNC( inq_varid )( ncid, "lat", &lat_id );
517  CHECK_EQUAL( 0, success );
518 
519  int lat_id_ref;
520  success = NCFUNC( inq_varid )( ncid_ref, "lat", &lat_id_ref );
521  CHECK_EQUAL( 0, success );
522 
523 #ifdef MOAB_HAVE_PNETCDF
524  // Enter independent I/O mode
525  success = NCFUNC( begin_indep_data )( ncid );
526  CHECK_EQUAL( 0, success );
527 #endif
528 
529 #ifdef MOAB_HAVE_PNETCDF
530  // Enter independent I/O mode
531  success = NCFUNC( begin_indep_data )( ncid_ref );
532  CHECK_EQUAL( 0, success );
533 #endif
534 
535  NCDF_SIZE start[] = { 0, 0, 0 };
536  NCDF_SIZE count[] = { 3, levels, 3458 };
537  const int size = 3 * levels * 3458;
538 
539  // Read variable T from output file
540  double T_vals[size];
541  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
542  CHECK_EQUAL( 0, success );
543 
544  // Read variable T from reference file
545  double T_vals_ref[size];
546  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
547  CHECK_EQUAL( 0, success );
548 
549  // Read variable lat from output file
550  count[0] = 3458;
551  double lat_vals[3458];
552  success = NCFUNC( get_vara_double )( ncid, lat_id, start, count, lat_vals );
553  CHECK_EQUAL( 0, success );
554 
555  // Read variable lat from reference file
556  double lat_vals_ref[3458];
557  success = NCFUNC( get_vara_double )( ncid_ref, lat_id_ref, start, count, lat_vals_ref );
558  CHECK_EQUAL( 0, success );
559 
560 #ifdef MOAB_HAVE_PNETCDF
561  // End independent I/O mode
562  success = NCFUNC( end_indep_data )( ncid );
563  CHECK_EQUAL( 0, success );
564 #endif
565 
566 #ifdef MOAB_HAVE_PNETCDF
567  // End independent I/O mode
568  success = NCFUNC( end_indep_data )( ncid_ref );
569  CHECK_EQUAL( 0, success );
570 #endif
571 
572  success = NCFUNC( close )( ncid );
573  CHECK_EQUAL( 0, success );
574 
575  success = NCFUNC( close )( ncid_ref );
576  CHECK_EQUAL( 0, success );
577 
578  // Check T values
579  for( int i = 0; i < size; i++ )
580  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
581 
582  // Check gw values
583  for( int i = 0; i < 3458; i++ )
584  CHECK_REAL_EQUAL( lat_vals_ref[i], lat_vals[i], eps );
585  }
586 }
587 
588 // Write vertex variable vorticity, edge variable u and cell variable ke
589 void test_mpas_read_write_vars()
590 {
591  int procs = 1;
592 #ifdef MOAB_HAVE_MPI
593  MPI_Comm_size( MPI_COMM_WORLD, &procs );
594 #endif
595 
596 // We will not test NC writer in parallel without pnetcdf support
597 #ifndef MOAB_HAVE_PNETCDF
598  if( procs > 1 ) return;
599 #endif
600 
601  Core moab;
602  Interface& mb = moab;
603 
604  std::string read_opts;
605  get_mpas_read_options( read_opts );
606 
607  EntityHandle set;
608  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
609 
610  // Read non-set variables vorticity, u and ke
611  read_opts += ";VARIABLE=vorticity,u,ke;DEBUG_IO=0";
612  if( procs > 1 ) read_opts += ";PARALLEL_RESOLVE_SHARED_ENTS";
613  rval = mb.load_file( example_mpas.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
614 
615  // Write variables vorticity, u and ke
616  std::string write_opts = ";;VARIABLE=vorticity,u,ke;DEBUG_IO=0";
617 #ifdef MOAB_HAVE_MPI
618  // Use parallel options
619  write_opts += ";PARALLEL=WRITE_PART";
620 #endif
621  if( procs > 1 )
622  rval = mb.write_file( "test_par_mpas_vars.nc", 0, write_opts.c_str(), &set, 1 );
623  else
624  rval = mb.write_file( "test_mpas_vars.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
625 }
626 
627 // Check vertex variable vorticity, edge variable u and cell variable ke
628 void test_mpas_check_vars()
629 {
630  int rank = 0;
631  int procs = 1;
632 #ifdef MOAB_HAVE_MPI
633  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
634  MPI_Comm_size( MPI_COMM_WORLD, &procs );
635 #endif
636 
637 // We will not test NC writer in parallel without pnetcdf support
638 #ifndef MOAB_HAVE_PNETCDF
639  if( procs > 1 ) return;
640 #endif
641 
642  if( 0 == rank )
643  {
644  int ncid;
645  int ncid_ref;
646  int success;
647 
648  std::string filename;
649  if( procs > 1 )
650  filename = "test_par_mpas_vars.nc";
651  else
652  filename = "test_mpas_vars.nc";
653 
654 #ifdef MOAB_HAVE_PNETCDF
655  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
656 #else
657  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
658 #endif
659  CHECK_EQUAL( 0, success );
660 
661 #ifdef MOAB_HAVE_PNETCDF
662  success = NCFUNC( open )( MPI_COMM_SELF, example_mpas.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
663 #else
664  success = NCFUNC( open )( example_mpas.c_str(), NC_NOWRITE, &ncid_ref );
665 #endif
666  CHECK_EQUAL( 0, success );
667 
668  int vorticity_id;
669  success = NCFUNC( inq_varid )( ncid, "vorticity", &vorticity_id );
670  CHECK_EQUAL( 0, success );
671 
672  int vorticity_id_ref;
673  success = NCFUNC( inq_varid )( ncid_ref, "vorticity", &vorticity_id_ref );
674  CHECK_EQUAL( 0, success );
675 
676  int u_id;
677  success = NCFUNC( inq_varid )( ncid, "u", &u_id );
678  CHECK_EQUAL( 0, success );
679 
680  int u_id_ref;
681  success = NCFUNC( inq_varid )( ncid_ref, "u", &u_id_ref );
682  CHECK_EQUAL( 0, success );
683 
684  int ke_id;
685  success = NCFUNC( inq_varid )( ncid, "ke", &ke_id );
686  CHECK_EQUAL( 0, success );
687 
688  int ke_id_ref;
689  success = NCFUNC( inq_varid )( ncid_ref, "ke", &ke_id_ref );
690  CHECK_EQUAL( 0, success );
691 
692 #ifdef MOAB_HAVE_PNETCDF
693  // Enter independent I/O mode
694  success = NCFUNC( begin_indep_data )( ncid );
695  CHECK_EQUAL( 0, success );
696 #endif
697 
698 #ifdef MOAB_HAVE_PNETCDF
699  // Enter independent I/O mode
700  success = NCFUNC( begin_indep_data )( ncid_ref );
701  CHECK_EQUAL( 0, success );
702 #endif
703 
704  NCDF_SIZE start[] = { 0, 0, 0 };
705  NCDF_SIZE count[] = { 2, 1, mpas_levels };
706  const int size1 = 2 * 1280 * mpas_levels;
707  const int size2 = 2 * 1920 * mpas_levels;
708  const int size3 = 2 * 642 * mpas_levels;
709 
710  // Read vertex variable vorticity from output file
711  count[1] = 1280;
712  double vorticity_vals[size1];
713  success = NCFUNC( get_vara_double )( ncid, vorticity_id, start, count, vorticity_vals );
714  CHECK_EQUAL( 0, success );
715 
716  // Read vertex variable vorticity from reference file
717  double vorticity_vals_ref[size1];
718  success = NCFUNC( get_vara_double )( ncid_ref, vorticity_id_ref, start, count, vorticity_vals_ref );
719  CHECK_EQUAL( 0, success );
720 
721  // Read edge variable u from output file
722  count[1] = 1920;
723  double u_vals[size2];
724  success = NCFUNC( get_vara_double )( ncid, u_id, start, count, u_vals );
725  CHECK_EQUAL( 0, success );
726 
727  // Read edge variable u from reference file
728  double u_vals_ref[size2];
729  success = NCFUNC( get_vara_double )( ncid_ref, u_id_ref, start, count, u_vals_ref );
730  CHECK_EQUAL( 0, success );
731 
732  // Read cell variable ke from output file
733  count[1] = 642;
734  double ke_vals[size3];
735  success = NCFUNC( get_vara_double )( ncid, ke_id, start, count, ke_vals );
736  CHECK_EQUAL( 0, success );
737 
738  // Read cell variable ke from reference file
739  double ke_vals_ref[size3];
740  success = NCFUNC( get_vara_double )( ncid_ref, ke_id_ref, start, count, ke_vals_ref );
741  CHECK_EQUAL( 0, success );
742 
743 #ifdef MOAB_HAVE_PNETCDF
744  // End independent I/O mode
745  success = NCFUNC( end_indep_data )( ncid );
746  CHECK_EQUAL( 0, success );
747 #endif
748 
749 #ifdef MOAB_HAVE_PNETCDF
750  // End independent I/O mode
751  success = NCFUNC( end_indep_data )( ncid_ref );
752  CHECK_EQUAL( 0, success );
753 #endif
754 
755  success = NCFUNC( close )( ncid );
756  CHECK_EQUAL( 0, success );
757 
758  success = NCFUNC( close )( ncid_ref );
759  CHECK_EQUAL( 0, success );
760 
761  // Check vorticity values
762  for( int i = 0; i < size1; i++ )
763  CHECK_REAL_EQUAL( vorticity_vals_ref[i], vorticity_vals[i], eps );
764 
765  // Check u values
766  for( int i = 0; i < size2; i++ )
767  CHECK_REAL_EQUAL( u_vals_ref[i], u_vals[i], eps );
768 
769  // Check ke values
770  for( int i = 0; i < size3; i++ )
771  CHECK_REAL_EQUAL( ke_vals_ref[i], ke_vals[i], eps );
772  }
773 }
774 
775 // Write vertex variable u, edge variable wind, cell variable vorticity (on layers),
776 // and cell variable pressure (on interfaces)
777 void test_gcrm_read_write_vars()
778 {
779  int procs = 1;
780 #ifdef MOAB_HAVE_MPI
781  MPI_Comm_size( MPI_COMM_WORLD, &procs );
782 #endif
783 
784 // We will not test NC writer in parallel without pnetcdf support
785 #ifndef MOAB_HAVE_PNETCDF
786  if( procs > 1 ) return;
787 #endif
788 
789  Core moab;
790  Interface& mb = moab;
791 
792  std::string read_opts;
793  // we can use the same base options as mpas, because the zoltan can apply too
794  get_mpas_read_options( read_opts );
795 
796  EntityHandle set;
797  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
798 
799  // Read non-set variables u, wind, vorticity and pressure
800  read_opts += ";VARIABLE=u,wind,vorticity,pressure;DEBUG_IO=0";
801  if( procs > 1 ) read_opts += ";PARALLEL_RESOLVE_SHARED_ENTS";
802  rval = mb.load_file( example_gcrm.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
803 
804  // Write variables u, wind, vorticity and pressure
805  std::string write_opts = ";;VARIABLE=u,wind,vorticity,pressure;DEBUG_IO=0";
806 #ifdef MOAB_HAVE_MPI
807  // Use parallel options
808  write_opts += ";PARALLEL=WRITE_PART";
809 #endif
810  if( procs > 1 )
811  rval = mb.write_file( "test_par_gcrm_vars.nc", 0, write_opts.c_str(), &set, 1 );
812  else
813  rval = mb.write_file( "test_gcrm_vars.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
814 }
815 
816 // Check vertex variable u, edge variable wind, cell variable vorticity (on layers),
817 // and cell variable pressure (on interfaces)
818 void test_gcrm_check_vars()
819 {
820  int rank = 0;
821  int procs = 1;
822 #ifdef MOAB_HAVE_MPI
823  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
824  MPI_Comm_size( MPI_COMM_WORLD, &procs );
825 #endif
826 
827 // We will not test NC writer in parallel without pnetcdf support
828 #ifndef MOAB_HAVE_PNETCDF
829  if( procs > 1 ) return;
830 #endif
831 
832  if( 0 == rank )
833  {
834  int ncid;
835  int ncid_ref;
836  int success;
837 
838  std::string filename;
839  if( procs > 1 )
840  filename = "test_par_gcrm_vars.nc";
841  else
842  filename = "test_gcrm_vars.nc";
843 
844 #ifdef MOAB_HAVE_PNETCDF
845  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
846 #else
847  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
848 #endif
849  CHECK_EQUAL( 0, success );
850 
851 #ifdef MOAB_HAVE_PNETCDF
852  success = NCFUNC( open )( MPI_COMM_SELF, example_gcrm.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
853 #else
854  success = NCFUNC( open )( example_gcrm.c_str(), NC_NOWRITE, &ncid_ref );
855 #endif
856  CHECK_EQUAL( 0, success );
857 
858  int u_id;
859  success = NCFUNC( inq_varid )( ncid, "u", &u_id );
860  CHECK_EQUAL( 0, success );
861 
862  int u_id_ref;
863  success = NCFUNC( inq_varid )( ncid_ref, "u", &u_id_ref );
864  CHECK_EQUAL( 0, success );
865 
866  int wind_id;
867  success = NCFUNC( inq_varid )( ncid, "wind", &wind_id );
868  CHECK_EQUAL( 0, success );
869 
870  int wind_id_ref;
871  success = NCFUNC( inq_varid )( ncid_ref, "wind", &wind_id_ref );
872  CHECK_EQUAL( 0, success );
873 
874  int vorticity_id;
875  success = NCFUNC( inq_varid )( ncid, "vorticity", &vorticity_id );
876  CHECK_EQUAL( 0, success );
877 
878  int vorticity_id_ref;
879  success = NCFUNC( inq_varid )( ncid_ref, "vorticity", &vorticity_id_ref );
880  CHECK_EQUAL( 0, success );
881 
882  int pressure_id;
883  success = NCFUNC( inq_varid )( ncid, "pressure", &pressure_id );
884  CHECK_EQUAL( 0, success );
885 
886  int pressure_id_ref;
887  success = NCFUNC( inq_varid )( ncid_ref, "pressure", &pressure_id_ref );
888  CHECK_EQUAL( 0, success );
889 
890 #ifdef MOAB_HAVE_PNETCDF
891  // Enter independent I/O mode
892  success = NCFUNC( begin_indep_data )( ncid );
893  CHECK_EQUAL( 0, success );
894 #endif
895 
896 #ifdef MOAB_HAVE_PNETCDF
897  // Enter independent I/O mode
898  success = NCFUNC( begin_indep_data )( ncid_ref );
899  CHECK_EQUAL( 0, success );
900 #endif
901 
902  NCDF_SIZE start[] = { 0, 0, 0 };
903  NCDF_SIZE count[] = { 2, 1, levels };
904  const int size1 = 2 * 1280 * levels;
905  const int size2 = 2 * 1920 * levels;
906  const int size3 = 2 * 642 * levels;
907 
908  // Read vertex variable u from output file
909  count[1] = 1280;
910  double u_vals[size1];
911  success = NCFUNC( get_vara_double )( ncid, u_id, start, count, u_vals );
912  CHECK_EQUAL( 0, success );
913 
914  // Read vertex variable u from reference file
915  double u_vals_ref[size1];
916  success = NCFUNC( get_vara_double )( ncid_ref, u_id_ref, start, count, u_vals_ref );
917  CHECK_EQUAL( 0, success );
918 
919  // Read edge variable wind from output file
920  count[1] = 1920;
921  double wind_vals[size2];
922  success = NCFUNC( get_vara_double )( ncid, wind_id, start, count, wind_vals );
923  CHECK_EQUAL( 0, success );
924 
925  // Read edge variable wind from reference file
926  double wind_vals_ref[size2];
927  success = NCFUNC( get_vara_double )( ncid_ref, wind_id_ref, start, count, wind_vals_ref );
928  CHECK_EQUAL( 0, success );
929 
930  // Read cell variable vorticity from output file
931  count[1] = 642;
932  double vorticity_vals[size3];
933  success = NCFUNC( get_vara_double )( ncid, vorticity_id, start, count, vorticity_vals );
934  CHECK_EQUAL( 0, success );
935 
936  // Read cell variable vorticity from reference file
937  double vorticity_vals_ref[size3];
938  success = NCFUNC( get_vara_double )( ncid_ref, vorticity_id_ref, start, count, vorticity_vals_ref );
939  CHECK_EQUAL( 0, success );
940 
941  // Read cell variable pressure from output file
942  double pressure_vals[size3];
943  success = NCFUNC( get_vara_double )( ncid, pressure_id, start, count, pressure_vals );
944  CHECK_EQUAL( 0, success );
945 
946  // Read cell variable pressure from reference file
947  double pressure_vals_ref[size3];
948  success = NCFUNC( get_vara_double )( ncid_ref, pressure_id_ref, start, count, pressure_vals_ref );
949  CHECK_EQUAL( 0, success );
950 
951 #ifdef MOAB_HAVE_PNETCDF
952  // End independent I/O mode
953  success = NCFUNC( end_indep_data )( ncid );
954  CHECK_EQUAL( 0, success );
955 #endif
956 
957 #ifdef MOAB_HAVE_PNETCDF
958  // End independent I/O mode
959  success = NCFUNC( end_indep_data )( ncid_ref );
960  CHECK_EQUAL( 0, success );
961 #endif
962 
963  success = NCFUNC( close )( ncid );
964  CHECK_EQUAL( 0, success );
965 
966  success = NCFUNC( close )( ncid_ref );
967  CHECK_EQUAL( 0, success );
968 
969  // Check u values
970  for( int i = 0; i < size1; i++ )
971  CHECK_REAL_EQUAL( u_vals_ref[i], u_vals[i], eps );
972 
973  // Check wind values
974  for( int i = 0; i < size2; i++ )
975  CHECK_REAL_EQUAL( wind_vals_ref[i], wind_vals[i], eps );
976 
977  // Check vorticity and pressure values
978  for( int i = 0; i < size3; i++ )
979  {
980  CHECK_REAL_EQUAL( vorticity_vals_ref[i], vorticity_vals[i], eps );
981  CHECK_REAL_EQUAL( pressure_vals_ref[i], pressure_vals[i], eps );
982  }
983  }
984 }
985 
986 // Read non-set variable T on all 3 timesteps, and write only timestep 2
987 void test_eul_read_write_timestep()
988 {
989  int procs = 1;
990 #ifdef MOAB_HAVE_MPI
991  MPI_Comm_size( MPI_COMM_WORLD, &procs );
992 #endif
993 
994 // We will not test NC writer in parallel without pnetcdf support
995 #ifndef MOAB_HAVE_PNETCDF
996  if( procs > 1 ) return;
997 #endif
998 
999  Core moab;
1000  Interface& mb = moab;
1001 
1002  std::string read_opts;
1003  get_eul_read_options( read_opts );
1004 
1005  EntityHandle set;
1006  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
1007 
1008  // Read non-set variable T
1009  read_opts += ";VARIABLE=T;DEBUG_IO=0";
1010  rval = mb.load_file( example_eul.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1011 
1012  // Write variable T on timestep 2
1013  std::string write_opts = ";;VARIABLE=T;TIMESTEP=2;DEBUG_IO=0";
1014 #ifdef MOAB_HAVE_MPI
1015  // Use parallel options
1016  write_opts += ";PARALLEL=WRITE_PART";
1017 #endif
1018  if( procs > 1 )
1019  rval = mb.write_file( "test_par_eul_T2.nc", 0, write_opts.c_str(), &set, 1 );
1020  else
1021  rval = mb.write_file( "test_eul_T2.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1022 }
1023 
1024 void test_eul_check_timestep()
1025 {
1026  int rank = 0;
1027  int procs = 1;
1028 #ifdef MOAB_HAVE_MPI
1029  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
1030  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1031 #endif
1032 
1033 // We will not test NC writer in parallel without pnetcdf support
1034 #ifndef MOAB_HAVE_PNETCDF
1035  if( procs > 1 ) return;
1036 #endif
1037 
1038  if( 0 == rank )
1039  {
1040  int ncid;
1041  int ncid_ref;
1042  int success;
1043 
1044  std::string filename;
1045  if( procs > 1 )
1046  filename = "test_par_eul_T2.nc";
1047  else
1048  filename = "test_eul_T2.nc";
1049 
1050 #ifdef MOAB_HAVE_PNETCDF
1051  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
1052 #else
1053  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
1054 #endif
1055  CHECK_EQUAL( 0, success );
1056 
1057 #ifdef MOAB_HAVE_PNETCDF
1058  success = NCFUNC( open )( MPI_COMM_SELF, example_eul.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
1059 #else
1060  success = NCFUNC( open )( example_eul.c_str(), NC_NOWRITE, &ncid_ref );
1061 #endif
1062  CHECK_EQUAL( 0, success );
1063 
1064  int T_id;
1065  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
1066  CHECK_EQUAL( 0, success );
1067 
1068  int T_id_ref;
1069  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
1070  CHECK_EQUAL( 0, success );
1071 
1072 #ifdef MOAB_HAVE_PNETCDF
1073  // Enter independent I/O mode
1074  success = NCFUNC( begin_indep_data )( ncid );
1075  CHECK_EQUAL( 0, success );
1076 #endif
1077 
1078 #ifdef MOAB_HAVE_PNETCDF
1079  // Enter independent I/O mode
1080  success = NCFUNC( begin_indep_data )( ncid_ref );
1081  CHECK_EQUAL( 0, success );
1082 #endif
1083 
1084  NCDF_SIZE start[] = { 0, 0, 0, 0 };
1085  NCDF_SIZE count[] = { 1, levels, 48, 96 };
1086  const int size = levels * 48 * 96;
1087 
1088  // Read variable T from output file (timestep 0)
1089  double T_vals[size];
1090  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
1091  CHECK_EQUAL( 0, success );
1092 
1093  // Read variable T from reference file (timestep 2)
1094  start[0] = 2;
1095  double T_vals_ref[size];
1096  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
1097  CHECK_EQUAL( 0, success );
1098 
1099 #ifdef MOAB_HAVE_PNETCDF
1100  // End independent I/O mode
1101  success = NCFUNC( end_indep_data )( ncid );
1102  CHECK_EQUAL( 0, success );
1103 #endif
1104 
1105 #ifdef MOAB_HAVE_PNETCDF
1106  // End independent I/O mode
1107  success = NCFUNC( end_indep_data )( ncid_ref );
1108  CHECK_EQUAL( 0, success );
1109 #endif
1110 
1111  success = NCFUNC( close )( ncid );
1112  CHECK_EQUAL( 0, success );
1113 
1114  success = NCFUNC( close )( ncid_ref );
1115  CHECK_EQUAL( 0, success );
1116 
1117  // Check T values
1118  for( int i = 0; i < size; i++ )
1119  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
1120  }
1121 }
1122 
1123 // Read non-set variables T, U and V
1124 // Write variable T, append U, and then append V (with a new name)
1125 void test_eul_read_write_append()
1126 {
1127  int procs = 1;
1128 #ifdef MOAB_HAVE_MPI
1129  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1130 #endif
1131 
1132 // We will not test NC writer in parallel without pnetcdf support
1133 #ifndef MOAB_HAVE_PNETCDF
1134  if( procs > 1 ) return;
1135 #endif
1136 
1137  Core moab;
1138  Interface& mb = moab;
1139 
1140  std::string read_opts;
1141  get_eul_read_options( read_opts );
1142 
1143  EntityHandle set;
1144  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
1145 
1146  // Load non-set variables T, U, V, and the mesh
1147  read_opts += ";VARIABLE=T,U,V;DEBUG_IO=0";
1148  rval = mb.load_file( example_eul.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1149 
1150  // Write variable T
1151  std::string write_opts = ";;VARIABLE=T;DEBUG_IO=0";
1152 #ifdef MOAB_HAVE_MPI
1153  // Use parallel options
1154  write_opts += ";PARALLEL=WRITE_PART";
1155 #endif
1156  if( procs > 1 )
1157  rval = mb.write_file( "test_par_eul_append.nc", 0, write_opts.c_str(), &set, 1 );
1158  else
1159  rval = mb.write_file( "test_eul_append.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1160 
1161  // Append to the file variable U
1162  write_opts = ";;VARIABLE=U;APPEND;DEBUG_IO=0";
1163 #ifdef MOAB_HAVE_MPI
1164  // Use parallel options
1165  write_opts += ";PARALLEL=WRITE_PART";
1166 #endif
1167  if( procs > 1 )
1168  rval = mb.write_file( "test_par_eul_append.nc", 0, write_opts.c_str(), &set, 1 );
1169  else
1170  rval = mb.write_file( "test_eul_append.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1171 
1172  // Append to the file variable V, renamed to VNEWNAME
1173  write_opts = ";;VARIABLE=V;RENAME=VNEWNAME;APPEND;DEBUG_IO=0";
1174 #ifdef MOAB_HAVE_MPI
1175  // Use parallel options
1176  write_opts += ";PARALLEL=WRITE_PART";
1177 #endif
1178  if( procs > 1 )
1179  rval = mb.write_file( "test_par_eul_append.nc", 0, write_opts.c_str(), &set, 1 );
1180  else
1181  rval = mb.write_file( "test_eul_append.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1182 }
1183 
1184 void test_eul_check_append()
1185 {
1186  int rank = 0;
1187  int procs = 1;
1188 #ifdef MOAB_HAVE_MPI
1189  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
1190  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1191 #endif
1192 
1193 // We will not test NC writer in parallel without pnetcdf support
1194 #ifndef MOAB_HAVE_PNETCDF
1195  if( procs > 1 ) return;
1196 #endif
1197 
1198  if( 0 == rank )
1199  {
1200  int ncid;
1201  int ncid_ref;
1202  int success;
1203 
1204  std::string filename;
1205  if( procs > 1 )
1206  filename = "test_par_eul_append.nc";
1207  else
1208  filename = "test_eul_append.nc";
1209 
1210 #ifdef MOAB_HAVE_PNETCDF
1211  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
1212 #else
1213  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
1214 #endif
1215  CHECK_EQUAL( 0, success );
1216 
1217 #ifdef MOAB_HAVE_PNETCDF
1218  success = NCFUNC( open )( MPI_COMM_SELF, example_eul.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
1219 #else
1220  success = NCFUNC( open )( example_eul.c_str(), NC_NOWRITE, &ncid_ref );
1221 #endif
1222  CHECK_EQUAL( 0, success );
1223 
1224  int T_id;
1225  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
1226  CHECK_EQUAL( 0, success );
1227 
1228  int T_id_ref;
1229  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
1230  CHECK_EQUAL( 0, success );
1231 
1232  int U_id;
1233  success = NCFUNC( inq_varid )( ncid, "U", &U_id );
1234  CHECK_EQUAL( 0, success );
1235 
1236  int U_id_ref;
1237  success = NCFUNC( inq_varid )( ncid_ref, "U", &U_id_ref );
1238  CHECK_EQUAL( 0, success );
1239 
1240  int V_id;
1241  success = NCFUNC( inq_varid )( ncid, "VNEWNAME", &V_id );
1242  CHECK_EQUAL( 0, success );
1243 
1244  int V_id_ref;
1245  success = NCFUNC( inq_varid )( ncid_ref, "V", &V_id_ref );
1246  CHECK_EQUAL( 0, success );
1247 
1248 #ifdef MOAB_HAVE_PNETCDF
1249  // Enter independent I/O mode
1250  success = NCFUNC( begin_indep_data )( ncid );
1251  CHECK_EQUAL( 0, success );
1252 #endif
1253 
1254 #ifdef MOAB_HAVE_PNETCDF
1255  // Enter independent I/O mode
1256  success = NCFUNC( begin_indep_data )( ncid_ref );
1257  CHECK_EQUAL( 0, success );
1258 #endif
1259 
1260  NCDF_SIZE start[] = { 0, 0, 0, 0 };
1261  NCDF_SIZE count[] = { 3, levels, 48, 96 };
1262  const int size = 3 * levels * 48 * 96;
1263 
1264  // Read variable T from output file
1265  double T_vals[size];
1266  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
1267  CHECK_EQUAL( 0, success );
1268 
1269  // Read variable T from reference file
1270  double T_vals_ref[size];
1271  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
1272  CHECK_EQUAL( 0, success );
1273 
1274  // Read variable U from output file
1275  double U_vals[size];
1276  success = NCFUNC( get_vara_double )( ncid, U_id, start, count, U_vals );
1277  CHECK_EQUAL( 0, success );
1278 
1279  // Read variable U from reference file
1280  double U_vals_ref[size];
1281  success = NCFUNC( get_vara_double )( ncid_ref, U_id_ref, start, count, U_vals_ref );
1282  CHECK_EQUAL( 0, success );
1283 
1284  // Read variable VNEWNAME from output file
1285  double V_vals[size];
1286  success = NCFUNC( get_vara_double )( ncid, V_id, start, count, V_vals );
1287  CHECK_EQUAL( 0, success );
1288 
1289  // Read variable V from reference file
1290  double V_vals_ref[size];
1291  success = NCFUNC( get_vara_double )( ncid_ref, V_id_ref, start, count, V_vals_ref );
1292  CHECK_EQUAL( 0, success );
1293 
1294 #ifdef MOAB_HAVE_PNETCDF
1295  // End independent I/O mode
1296  success = NCFUNC( end_indep_data )( ncid );
1297  CHECK_EQUAL( 0, success );
1298 #endif
1299 
1300 #ifdef MOAB_HAVE_PNETCDF
1301  // End independent I/O mode
1302  success = NCFUNC( end_indep_data )( ncid_ref );
1303  CHECK_EQUAL( 0, success );
1304 #endif
1305 
1306  success = NCFUNC( close )( ncid );
1307  CHECK_EQUAL( 0, success );
1308 
1309  success = NCFUNC( close )( ncid_ref );
1310  CHECK_EQUAL( 0, success );
1311 
1312  // Check T, U, and V values
1313  for( int i = 0; i < size; i++ )
1314  {
1315  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
1316  CHECK_REAL_EQUAL( U_vals_ref[i], U_vals[i], eps );
1317  CHECK_REAL_EQUAL( V_vals_ref[i], V_vals[i], eps );
1318  }
1319  }
1320 }
1321 
1322 void test_eul_read_write_across_files()
1323 {
1324  int procs = 1;
1325 #ifdef MOAB_HAVE_MPI
1326  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1327 #endif
1328 
1329 // We will not test NC writer in parallel without pnetcdf support
1330 #ifndef MOAB_HAVE_PNETCDF
1331  if( procs > 1 ) return;
1332 #endif
1333 
1334  Core moab;
1335  Interface& mb = moab;
1336 
1337  std::string read_opts;
1338 
1339  EntityHandle set;
1340  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
1341 
1342  // This file contains single timestep 2 (option TIMESTEP=0 will be implicitly used)
1343  // Read T as tag T2 with option TIMESTEPBASE=2
1344  get_eul_read_options( read_opts );
1345  read_opts += ";VARIABLE=T;TIMESTEPBASE=2;DEBUG_IO=0";
1346  rval = mb.load_file( example_eul_t2.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1347 
1348  // This file contains single timestep 0 (option TIMESTEP=0 will be implicitly used)
1349  // Read T as tag T0 with option TIMESTEPBASE=0
1350  get_eul_read_options( read_opts );
1351  read_opts += ";VARIABLE=T;TIMESTEPBASE=0;NOMESH;DEBUG_IO=0";
1352  rval = mb.load_file( example_eul_t0.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1353 
1354  // This file contains single timestep 1 (option TIMESTEP=0 will be implicitly used)
1355  // Read T as tag T1 with option TIMESTEPBASE=1
1356  get_eul_read_options( read_opts );
1357  read_opts += ";VARIABLE=T;TIMESTEPBASE=1;NOMESH;DEBUG_IO=0";
1358  rval = mb.load_file( example_eul_t1.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1359 
1360  // Write variable T with 3 timesteps
1361  std::string write_opts = ";;VARIABLE=T;TIMESTEP=0,1,2;DEBUG_IO=0";
1362 #ifdef MOAB_HAVE_MPI
1363  // Use parallel options
1364  write_opts += ";PARALLEL=WRITE_PART";
1365 #endif
1366  if( procs > 1 )
1367  rval = mb.write_file( "test_par_eul_across_files.nc", 0, write_opts.c_str(), &set, 1 );
1368  else
1369  rval = mb.write_file( "test_eul_across_files.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1370 }
1371 
1372 void test_eul_check_across_files()
1373 {
1374  int rank = 0;
1375  int procs = 1;
1376 #ifdef MOAB_HAVE_MPI
1377  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
1378  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1379 #endif
1380 
1381 // We will not test NC writer in parallel without pnetcdf support
1382 #ifndef MOAB_HAVE_PNETCDF
1383  if( procs > 1 ) return;
1384 #endif
1385 
1386  if( 0 == rank )
1387  {
1388  int ncid;
1389  int ncid_ref;
1390  int success;
1391 
1392  std::string filename;
1393  if( procs > 1 )
1394  filename = "test_par_eul_across_files.nc";
1395  else
1396  filename = "test_eul_across_files.nc";
1397 
1398 #ifdef MOAB_HAVE_PNETCDF
1399  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
1400 #else
1401  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
1402 #endif
1403  CHECK_EQUAL( 0, success );
1404 
1405 #ifdef MOAB_HAVE_PNETCDF
1406  success = NCFUNC( open )( MPI_COMM_SELF, example_eul.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
1407 #else
1408  success = NCFUNC( open )( example_eul.c_str(), NC_NOWRITE, &ncid_ref );
1409 #endif
1410  CHECK_EQUAL( 0, success );
1411 
1412  int T_id;
1413  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
1414  CHECK_EQUAL( 0, success );
1415 
1416  int T_id_ref;
1417  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
1418  CHECK_EQUAL( 0, success );
1419 
1420 #ifdef MOAB_HAVE_PNETCDF
1421  // Enter independent I/O mode
1422  success = NCFUNC( begin_indep_data )( ncid );
1423  CHECK_EQUAL( 0, success );
1424 #endif
1425 
1426 #ifdef MOAB_HAVE_PNETCDF
1427  // Enter independent I/O mode
1428  success = NCFUNC( begin_indep_data )( ncid_ref );
1429  CHECK_EQUAL( 0, success );
1430 #endif
1431 
1432  NCDF_SIZE start[] = { 0, 0, 0, 0 };
1433  NCDF_SIZE count[] = { 3, levels, 48, 96 };
1434  const int size = 3 * levels * 48 * 96;
1435 
1436  // Read variable T from output file (with 3 timesteps)
1437  double T_vals[size];
1438  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
1439  CHECK_EQUAL( 0, success );
1440 
1441  // Read variable T from reference file (with 3 timesteps)
1442  double T_vals_ref[size];
1443  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
1444  CHECK_EQUAL( 0, success );
1445 
1446 #ifdef MOAB_HAVE_PNETCDF
1447  // End independent I/O mode
1448  success = NCFUNC( end_indep_data )( ncid );
1449  CHECK_EQUAL( 0, success );
1450 #endif
1451 
1452 #ifdef MOAB_HAVE_PNETCDF
1453  // End independent I/O mode
1454  success = NCFUNC( end_indep_data )( ncid_ref );
1455  CHECK_EQUAL( 0, success );
1456 #endif
1457 
1458  success = NCFUNC( close )( ncid );
1459  CHECK_EQUAL( 0, success );
1460 
1461  success = NCFUNC( close )( ncid_ref );
1462  CHECK_EQUAL( 0, success );
1463 
1464  // Check T values
1465  for( int i = 0; i < size; i++ )
1466  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
1467  }
1468 }
1469 
1470 #ifdef MOAB_HAVE_MPI
1471 // NC writer should filter entities that are not owned, e.g. ghosted elements
1472 void test_eul_read_write_ghosting()
1473 {
1474  int procs = 1;
1475  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1476 
1477 // We will not test NC writer in parallel without pnetcdf support
1478 #ifndef MOAB_HAVE_PNETCDF
1479  if( procs > 1 ) return;
1480 #endif
1481 
1482  Core moab;
1483  Interface& mb = moab;
1484 
1485  EntityHandle set;
1486  ErrorCode rval = mb.create_meshset( MESHSET_SET, set );CHECK_ERR( rval );
1487 
1488  std::string read_opts = "PARALLEL=READ_PART;PARTITION;PARALLEL_RESOLVE_SHARED_ENTS;PARALLEL_"
1489  "GHOSTS=2.0.1;PARTITION_METHOD=SQIJ;VARIABLE=";
1490  rval = mb.load_file( example_eul.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1491 
1492  read_opts = "PARALLEL=READ_PART;PARTITION;PARTITION_METHOD=SQIJ;VARIABLE=T;NOMESH";
1493  rval = mb.load_file( example_eul.c_str(), &set, read_opts.c_str() );CHECK_ERR( rval );
1494 
1495  // Write variable T
1496  std::string write_opts = ";;PARALLEL=WRITE_PART;VARIABLE=T;DEBUG_IO=0";
1497  if( procs > 1 )
1498  rval = mb.write_file( "test_par_eul_ghosting.nc", 0, write_opts.c_str(), &set, 1 );
1499  else
1500  rval = mb.write_file( "test_eul_ghosting.nc", 0, write_opts.c_str(), &set, 1 );CHECK_ERR( rval );
1501 }
1502 
1503 void test_eul_check_ghosting()
1504 {
1505  int rank = 0;
1506  int procs = 1;
1507 #ifdef MOAB_HAVE_MPI
1508  MPI_Comm_rank( MPI_COMM_WORLD, &rank );
1509  MPI_Comm_size( MPI_COMM_WORLD, &procs );
1510 #endif
1511 
1512 // We will not test NC writer in parallel without pnetcdf support
1513 #ifndef MOAB_HAVE_PNETCDF
1514  if( procs > 1 ) return;
1515 #endif
1516 
1517  if( 0 == rank )
1518  {
1519  int ncid;
1520  int ncid_ref;
1521  int success;
1522 
1523  std::string filename;
1524  if( procs > 1 )
1525  filename = "test_par_eul_ghosting.nc";
1526  else
1527  filename = "test_eul_ghosting.nc";
1528 
1529 #ifdef MOAB_HAVE_PNETCDF
1530  success = NCFUNC( open )( MPI_COMM_SELF, filename.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid );
1531 #else
1532  success = NCFUNC( open )( filename.c_str(), NC_NOWRITE, &ncid );
1533 #endif
1534  CHECK_EQUAL( 0, success );
1535 
1536 #ifdef MOAB_HAVE_PNETCDF
1537  success = NCFUNC( open )( MPI_COMM_SELF, example_eul.c_str(), NC_NOWRITE, MPI_INFO_NULL, &ncid_ref );
1538 #else
1539  success = NCFUNC( open )( example_eul.c_str(), NC_NOWRITE, &ncid_ref );
1540 #endif
1541  CHECK_EQUAL( 0, success );
1542 
1543  int T_id;
1544  success = NCFUNC( inq_varid )( ncid, "T", &T_id );
1545  CHECK_EQUAL( 0, success );
1546 
1547  int T_id_ref;
1548  success = NCFUNC( inq_varid )( ncid_ref, "T", &T_id_ref );
1549  CHECK_EQUAL( 0, success );
1550 
1551 #ifdef MOAB_HAVE_PNETCDF
1552  // Enter independent I/O mode
1553  success = NCFUNC( begin_indep_data )( ncid );
1554  CHECK_EQUAL( 0, success );
1555 #endif
1556 
1557 #ifdef MOAB_HAVE_PNETCDF
1558  // Enter independent I/O mode
1559  success = NCFUNC( begin_indep_data )( ncid_ref );
1560  CHECK_EQUAL( 0, success );
1561 #endif
1562 
1563  NCDF_SIZE start[] = { 0, 0, 0, 0 };
1564  NCDF_SIZE count[] = { 3, levels, 48, 96 };
1565  const int size = 3 * levels * 48 * 96;
1566 
1567  // Read variable T from output file
1568  double T_vals[size];
1569  success = NCFUNC( get_vara_double )( ncid, T_id, start, count, T_vals );
1570  CHECK_EQUAL( 0, success );
1571 
1572  // Read variable T from reference file
1573  double T_vals_ref[size];
1574  success = NCFUNC( get_vara_double )( ncid_ref, T_id_ref, start, count, T_vals_ref );
1575  CHECK_EQUAL( 0, success );
1576 
1577 #ifdef MOAB_HAVE_PNETCDF
1578  // End independent I/O mode
1579  success = NCFUNC( end_indep_data )( ncid );
1580  CHECK_EQUAL( 0, success );
1581 #endif
1582 
1583 #ifdef MOAB_HAVE_PNETCDF
1584  // End independent I/O mode
1585  success = NCFUNC( end_indep_data )( ncid_ref );
1586  CHECK_EQUAL( 0, success );
1587 #endif
1588 
1589  success = NCFUNC( close )( ncid );
1590  CHECK_EQUAL( 0, success );
1591 
1592  success = NCFUNC( close )( ncid_ref );
1593  CHECK_EQUAL( 0, success );
1594 
1595  // Check T values
1596  for( int i = 0; i < size; i++ )
1597  CHECK_REAL_EQUAL( T_vals_ref[i], T_vals[i], eps );
1598  }
1599 }
1600 #endif
1601 
1602 void get_eul_read_options( std::string& opts )
1603 {
1604 #ifdef MOAB_HAVE_MPI
1605  // Use parallel options
1606  opts = ";;PARALLEL=READ_PART;PARTITION_METHOD=SQIJ";
1607 #else
1608  opts = ";;";
1609 #endif
1610 }
1611 
1612 void get_fv_read_options( std::string& opts )
1613 {
1614 #ifdef MOAB_HAVE_MPI
1615  // Use parallel options
1616  opts = ";;PARALLEL=READ_PART;PARTITION_METHOD=SQIJ";
1617 #else
1618  opts = ";;";
1619 #endif
1620 }
1621 
1622 void get_homme_read_options( std::string& opts )
1623 {
1624 #ifdef MOAB_HAVE_MPI
1625  // Use parallel options
1626  opts = ";;PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL";
1627 #else
1628  opts = ";;";
1629 #endif
1630 }
1631 
1632 void get_mpas_read_options( std::string& opts )
1633 {
1634 #ifdef MOAB_HAVE_MPI
1635  // Use parallel options
1636 #ifdef MOAB_HAVE_ZOLTAN
1637  opts = ";;PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN";
1638 #else
1639  opts = ";;PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL";
1640 #endif
1641 #else
1642  opts = ";;";
1643 #endif
1644 }