MOAB: Mesh Oriented datABase  (version 5.5.0)
mpastrvpart.cpp
Go to the documentation of this file.
1 #include "TestUtil.hpp"
2 #include "moab/Core.hpp"
3 #include "moab/ParallelComm.hpp"
4 #include "moab/ProgOptions.hpp"
5 #include "MBParallelConventions.h"
6 #include "moab/ReadUtilIface.hpp"
7 #include "MBTagConventions.hpp"
8 
9 #include <sstream>
10 
11 using namespace moab;
12 
13 std::string example = TestDir + "unittest/io/mpasx1.642.t.2.nc";
14 
15 void test_read_onevar_trivial();
16 void test_read_onevar_trivial_no_mixed_elements();
17 #if defined( MOAB_HAVE_MPI ) && defined( MOAB_HAVE_ZOLTAN )
18 void test_read_onevar_rcbzoltan();
19 void test_read_onevar_rcbzoltan_no_mixed_elements();
20 #endif
21 
22 void test_read_mesh_parallel_trivial();
23 void test_read_mesh_parallel_trivial_no_mixed_elements();
24 #if defined( MOAB_HAVE_MPI ) && defined( MOAB_HAVE_ZOLTAN )
25 void test_read_mesh_parallel_rcbzoltan();
26 void test_read_mesh_parallel_rcbzoltan_no_mixed_elements();
27 #endif
28 
29 void test_gather_onevar_on_rank0();
30 void test_gather_onevar_on_rank1();
31 
32 void test_multiple_loads_of_same_file();
33 void test_multiple_loads_of_same_file_no_mixed_elements();
34 
35 // Helper functions
36 void read_one_cell_var( bool rcbzoltan, bool no_mixed_elements );
37 void read_mesh_parallel( bool rcbzoltan, bool no_mixed_elements );
38 void gather_one_cell_var( int gather_set_rank );
39 void multiple_loads_of_same_file( bool no_mixed_elements );
40 
41 std::string read_options;
42 const double eps = 1e-20;
43 
44 int main( int argc, char* argv[] )
45 {
46  MPI_Init( &argc, &argv );
47  int result = 0;
48 
49  result += RUN_TEST( test_read_onevar_trivial );
50  result += RUN_TEST( test_read_onevar_trivial_no_mixed_elements );
51 #if defined( MOAB_HAVE_MPI ) && defined( MOAB_HAVE_ZOLTAN )
52  result += RUN_TEST( test_read_onevar_rcbzoltan );
53  result += RUN_TEST( test_read_onevar_rcbzoltan_no_mixed_elements );
54 #endif
55 
56  result += RUN_TEST( test_read_mesh_parallel_trivial );
57  result += RUN_TEST( test_read_mesh_parallel_trivial_no_mixed_elements );
58 #if defined( MOAB_HAVE_MPI ) && defined( MOAB_HAVE_ZOLTAN )
59  result += RUN_TEST( test_read_mesh_parallel_rcbzoltan );
60  result += RUN_TEST( test_read_mesh_parallel_rcbzoltan_no_mixed_elements );
61 #endif
62 
63  result += RUN_TEST( test_gather_onevar_on_rank0 );
64  result += RUN_TEST( test_gather_onevar_on_rank1 );
65 
66  result += RUN_TEST( test_multiple_loads_of_same_file );
67  result += RUN_TEST( test_multiple_loads_of_same_file_no_mixed_elements );
68 
69  MPI_Finalize();
70  return result;
71 }
72 
73 void test_read_onevar_trivial()
74 {
75  read_one_cell_var( false, false );
76 }
77 
78 void test_read_onevar_trivial_no_mixed_elements()
79 {
80  read_one_cell_var( false, true );
81 }
82 
83 void test_read_onevar_rcbzoltan()
84 {
85  read_one_cell_var( true, false );
86 }
87 
88 void test_read_onevar_rcbzoltan_no_mixed_elements()
89 {
90  read_one_cell_var( true, true );
91 }
92 
93 void test_read_mesh_parallel_trivial()
94 {
95  read_mesh_parallel( false, false );
96 }
97 
98 void test_read_mesh_parallel_trivial_no_mixed_elements()
99 {
100  read_mesh_parallel( false, true );
101 }
102 
103 void test_read_mesh_parallel_rcbzoltan()
104 {
105  read_mesh_parallel( true, false );
106 }
107 
108 void test_read_mesh_parallel_rcbzoltan_no_mixed_elements()
109 {
110  read_mesh_parallel( true, true );
111 }
112 
113 void test_gather_onevar_on_rank0()
114 {
115  gather_one_cell_var( 0 );
116 }
117 
118 void test_gather_onevar_on_rank1()
119 {
120  gather_one_cell_var( 1 );
121 }
122 
123 void test_multiple_loads_of_same_file()
124 {
125  multiple_loads_of_same_file( false );
126 }
127 
128 void test_multiple_loads_of_same_file_no_mixed_elements()
129 {
130  multiple_loads_of_same_file( true );
131 }
132 
133 // Helper functions
134 void read_one_cell_var( bool rcbzoltan, bool no_mixed_elements )
135 {
136  Core moab;
137  Interface& mb = moab;
138 
139  read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;NO_EDGES;VARIABLE=ke";
140  if( rcbzoltan ) read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;NO_EDGES;VARIABLE=ke";
141 
142  if( no_mixed_elements ) read_options += ";NO_MIXED_ELEMENTS";
143 
144  ErrorCode rval = mb.load_file( example.c_str(), NULL, read_options.c_str() );CHECK_ERR( rval );
145 
146  // Get local edges
147  Range local_edges;
148  rval = mb.get_entities_by_type( 0, MBEDGE, local_edges );CHECK_ERR( rval );
149  CHECK_EQUAL( (size_t)0, local_edges.size() );
150 
151  // Get local cells
152  Range local_cells;
153  rval = mb.get_entities_by_type( 0, MBPOLYGON, local_cells );CHECK_ERR( rval );
154 
155  Tag gid_tag = mb.globalId_tag();
156 
157  std::vector< int > gids( local_cells.size() );
158  rval = mb.tag_get_data( gid_tag, local_cells, &gids[0] );
159  Range local_cell_gids;
160  std::copy( gids.rbegin(), gids.rend(), range_inserter( local_cell_gids ) );
161 
162  ParallelComm* pcomm = ParallelComm::get_pcomm( &mb, 0 );
163  int procs = pcomm->proc_config().proc_size();
164  int rank = pcomm->proc_config().proc_rank();
165 
166  // Make check runs this test on two processors
167  if( 2 == procs )
168  {
169  CHECK_EQUAL( (size_t)321, local_cells.size() );
170  CHECK_EQUAL( (size_t)321, local_cell_gids.size() );
171 
172  // Check tag for cell variable ke at timestep 0
173  Tag ke_tag0;
174  rval = mb.tag_get_handle( "ke0", 1, MB_TYPE_DOUBLE, ke_tag0 );CHECK_ERR( rval );
175 
176  // Check tag for cell variable ke at timestep 1
177  Tag ke_tag1;
178  rval = mb.tag_get_handle( "ke1", 1, MB_TYPE_DOUBLE, ke_tag1 );CHECK_ERR( rval );
179 
180  // Get ke0 and ke1 tag values on 3 local cells
181  EntityHandle cell_ents[] = { local_cells[0], local_cells[160], local_cells[320] };
182  double ke0_val[3];
183  rval = mb.tag_get_data( ke_tag0, cell_ents, 3, ke0_val );CHECK_ERR( rval );
184  double ke1_val[3];
185  rval = mb.tag_get_data( ke_tag1, cell_ents, 3, ke1_val );CHECK_ERR( rval );
186 
187  if( rcbzoltan )
188  {
189 
190  CHECK_EQUAL( (size_t)1, local_cells.psize() );
191  CHECK_EQUAL( (size_t)37, local_cell_gids.psize() );
192 
193  if( 0 == rank )
194  {
195  CHECK_EQUAL( 1, (int)local_cell_gids[0] );
196  CHECK_EQUAL( 284, (int)local_cell_gids[160] );
197  CHECK_EQUAL( 630, (int)local_cell_gids[320] );
198 
199  CHECK_REAL_EQUAL( 15.001, ke0_val[0], eps );
200  CHECK_REAL_EQUAL( 16.284, ke0_val[1], eps );
201  CHECK_REAL_EQUAL( 16.630, ke0_val[2], eps );
202  CHECK_REAL_EQUAL( 25.001, ke1_val[0], eps );
203  CHECK_REAL_EQUAL( 26.284, ke1_val[1], eps );
204  CHECK_REAL_EQUAL( 26.630, ke1_val[2], eps );
205  }
206  else if( 1 == rank )
207  {
208  CHECK_EQUAL( 4, (int)local_cell_gids[0] );
209  CHECK_EQUAL( 341, (int)local_cell_gids[160] );
210  CHECK_EQUAL( 642, (int)local_cell_gids[320] );
211 
212  CHECK_REAL_EQUAL( 15.004, ke0_val[0], eps );
213  CHECK_REAL_EQUAL( 16.341, ke0_val[1], eps );
214  CHECK_REAL_EQUAL( 16.642, ke0_val[2], eps );
215  CHECK_REAL_EQUAL( 25.004, ke1_val[0], eps );
216  CHECK_REAL_EQUAL( 26.341, ke1_val[1], eps );
217  CHECK_REAL_EQUAL( 26.642, ke1_val[2], eps );
218  }
219  }
220  else
221  {
222  CHECK_EQUAL( (size_t)1, local_cell_gids.psize() );
223 
224  if( 0 == rank )
225  {
226  CHECK_EQUAL( (size_t)1, local_cells.psize() );
227  CHECK_EQUAL( 1, (int)local_cell_gids[0] );
228  CHECK_EQUAL( 161, (int)local_cell_gids[160] );
229  CHECK_EQUAL( 321, (int)local_cell_gids[320] );
230 
231  CHECK_REAL_EQUAL( 15.001, ke0_val[0], eps );
232  CHECK_REAL_EQUAL( 16.161, ke0_val[1], eps );
233  CHECK_REAL_EQUAL( 16.321, ke0_val[2], eps );
234  CHECK_REAL_EQUAL( 25.001, ke1_val[0], eps );
235  CHECK_REAL_EQUAL( 26.161, ke1_val[1], eps );
236  CHECK_REAL_EQUAL( 26.321, ke1_val[2], eps );
237  }
238  else if( 1 == rank )
239  {
240  CHECK_EQUAL( (size_t)1, local_cells.psize() );
241  CHECK_EQUAL( 322, (int)local_cell_gids[0] );
242  CHECK_EQUAL( 482, (int)local_cell_gids[160] );
243  CHECK_EQUAL( 642, (int)local_cell_gids[320] );
244 
245  CHECK_REAL_EQUAL( 16.322, ke0_val[0], eps );
246  CHECK_REAL_EQUAL( 16.482, ke0_val[1], eps );
247  CHECK_REAL_EQUAL( 16.642, ke0_val[2], eps );
248  CHECK_REAL_EQUAL( 26.322, ke1_val[0], eps );
249  CHECK_REAL_EQUAL( 26.482, ke1_val[1], eps );
250  CHECK_REAL_EQUAL( 26.642, ke1_val[2], eps );
251  }
252  }
253  }
254 }
255 
256 void read_mesh_parallel( bool rcbzoltan, bool no_mixed_elements )
257 {
258  Core moab;
259  Interface& mb = moab;
260 
261  read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
262  if( rcbzoltan )
263  read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
264 
265  if( no_mixed_elements ) read_options += ";NO_MIXED_ELEMENTS";
266 
267  ErrorCode rval = mb.load_file( example.c_str(), NULL, read_options.c_str() );CHECK_ERR( rval );
268 
269  ParallelComm* pcomm = ParallelComm::get_pcomm( &mb, 0 );
270  int procs = pcomm->proc_config().proc_size();
271  int rank = pcomm->proc_config().proc_rank();
272 
273  rval = pcomm->check_all_shared_handles();CHECK_ERR( rval );
274 
275  // Get local vertices
276  Range local_verts;
277  rval = mb.get_entities_by_type( 0, MBVERTEX, local_verts );CHECK_ERR( rval );
278 
279  int verts_num = local_verts.size();
280  if( 2 == procs )
281  {
282  if( rcbzoltan )
283  {
284  if( 0 == rank )
285  CHECK_EQUAL( 685, verts_num );
286  else if( 1 == rank )
287  CHECK_EQUAL( 685, verts_num ); // Not owned vertices included
288  }
289  else
290  {
291  if( 0 == rank )
292  CHECK_EQUAL( 1120, verts_num );
293  else if( 1 == rank )
294  CHECK_EQUAL( 1122, verts_num ); // Not owned vertices included
295  }
296  }
297 
298  rval = pcomm->filter_pstatus( local_verts, PSTATUS_NOT_OWNED, PSTATUS_NOT );CHECK_ERR( rval );
299 
300  verts_num = local_verts.size();
301  if( 2 == procs )
302  {
303  if( rcbzoltan )
304  {
305  if( 0 == rank )
306  CHECK_EQUAL( 685, verts_num );
307  else if( 1 == rank )
308  CHECK_EQUAL( 595, verts_num ); // Not owned vertices excluded
309  }
310  else
311  {
312  if( 0 == rank )
313  CHECK_EQUAL( 1120, verts_num );
314  else if( 1 == rank )
315  CHECK_EQUAL( 160, verts_num ); // Not owned vertices excluded
316  }
317  }
318 
319  // Get local edges
320  Range local_edges;
321  rval = mb.get_entities_by_type( 0, MBEDGE, local_edges );CHECK_ERR( rval );
322 
323  int edges_num = local_edges.size();
324  if( 2 == procs )
325  {
326  if( rcbzoltan )
327  {
328  if( 0 == rank )
329  CHECK_EQUAL( 1005, edges_num );
330  else if( 1 == rank )
331  CHECK_EQUAL( 1005, edges_num ); // Not owned edges included
332  }
333  else
334  {
335  if( 0 == rank )
336  CHECK_EQUAL( 1438, edges_num );
337  else if( 1 == rank )
338  CHECK_EQUAL( 1444, edges_num ); // Not owned edges included
339  }
340  }
341 
342  rval = pcomm->filter_pstatus( local_edges, PSTATUS_NOT_OWNED, PSTATUS_NOT );CHECK_ERR( rval );
343 
344  edges_num = local_edges.size();
345  if( 2 == procs )
346  {
347  if( rcbzoltan )
348  {
349  if( 0 == rank )
350  CHECK_EQUAL( 1005, edges_num );
351  else if( 1 == rank )
352  CHECK_EQUAL( 915, edges_num ); // Not owned edges excluded
353  }
354  else
355  {
356  if( 0 == rank )
357  CHECK_EQUAL( 1438, edges_num );
358  else if( 1 == rank )
359  CHECK_EQUAL( 482, edges_num ); // Not owned edges excluded
360  }
361  }
362 
363  // Get local cells
364  Range local_cells;
365  rval = mb.get_entities_by_type( 0, MBPOLYGON, local_cells );CHECK_ERR( rval );
366 
367  int cells_num = local_cells.size();
368  if( 2 == procs )
369  {
370  CHECK_EQUAL( 321, cells_num );
371  CHECK_EQUAL( (size_t)1, local_cells.psize() );
372  }
373 
374  rval = pcomm->filter_pstatus( local_cells, PSTATUS_NOT_OWNED, PSTATUS_NOT );CHECK_ERR( rval );
375 
376  cells_num = local_cells.size();
377  if( 2 == procs )
378  {
379  CHECK_EQUAL( 321, cells_num );
380  CHECK_EQUAL( (size_t)1, local_cells.psize() );
381  }
382 
383  std::cout << "proc: " << rank << " verts:" << verts_num << "\n";
384 
385  int total_verts_num;
386  MPI_Reduce( &verts_num, &total_verts_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm() );
387  if( 0 == rank )
388  {
389  std::cout << "total vertices: " << total_verts_num << "\n";
390  CHECK_EQUAL( 1280, total_verts_num );
391  }
392 
393  std::cout << "proc: " << rank << " edges:" << edges_num << "\n";
394 
395  int total_edges_num;
396  MPI_Reduce( &edges_num, &total_edges_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm() );
397  if( 0 == rank )
398  {
399  std::cout << "total edges: " << total_edges_num << "\n";
400  CHECK_EQUAL( 1920, total_edges_num );
401  }
402 
403  std::cout << "proc: " << rank << " cells:" << cells_num << "\n";
404 
405  int total_cells_num;
406  MPI_Reduce( &cells_num, &total_cells_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm() );
407  if( 0 == rank )
408  {
409  std::cout << "total cells: " << total_cells_num << "\n";
410  CHECK_EQUAL( 642, total_cells_num );
411  }
412 
413 #ifdef MOAB_HAVE_HDF5_PARALLEL
414  std::string write_options( "PARALLEL=WRITE_PART;" );
415 
416  std::string output_file = "test_mpas";
417  if( rcbzoltan ) output_file += "_rcbzoltan";
418  if( no_mixed_elements ) output_file += "_no_mixed_elements";
419  output_file += ".h5m";
420 
421  mb.write_file( output_file.c_str(), NULL, write_options.c_str() );
422 #endif
423 }
424 
425 void gather_one_cell_var( int gather_set_rank )
426 {
427  Core moab;
428  Interface& mb = moab;
429 
430  EntityHandle file_set;
431  ErrorCode rval = mb.create_meshset( MESHSET_SET, file_set );CHECK_ERR( rval );
432 
433  read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS";
434  std::ostringstream gather_set_option;
435  gather_set_option << ";GATHER_SET=" << gather_set_rank;
436  read_options += gather_set_option.str();
437 
438  rval = mb.load_file( example.c_str(), &file_set, read_options.c_str() );CHECK_ERR( rval );
439 
440  ParallelComm* pcomm = ParallelComm::get_pcomm( &mb, 0 );
441  int procs = pcomm->proc_config().proc_size();
442  int rank = pcomm->proc_config().proc_rank();
443 
444  // Make sure gather_set_rank is valid
445  if( gather_set_rank < 0 || gather_set_rank >= procs ) return;
446 
447  Range cells, cells_owned;
448  rval = mb.get_entities_by_type( file_set, MBPOLYGON, cells );CHECK_ERR( rval );
449 
450  // Get local owned cells
451  rval = pcomm->filter_pstatus( cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned );CHECK_ERR( rval );
452 
453  EntityHandle gather_set = 0;
454  if( gather_set_rank == rank )
455  {
456  // Get gather set
457  ReadUtilIface* readUtilIface;
458  mb.query_interface( readUtilIface );
459  rval = readUtilIface->get_gather_set( gather_set );CHECK_ERR( rval );
460  assert( gather_set != 0 );
461  }
462 
463  Tag ke_tag0, gid_tag;
464  rval = mb.tag_get_handle( "ke0", 1, MB_TYPE_DOUBLE, ke_tag0, MB_TAG_DENSE );CHECK_ERR( rval );
465 
466  gid_tag = mb.globalId_tag();
467 
468  pcomm->gather_data( cells_owned, ke_tag0, gid_tag, gather_set, gather_set_rank );
469 
470  if( gather_set_rank == rank )
471  {
472  // Get gather set cells
473  Range gather_set_cells;
474  rval = mb.get_entities_by_type( gather_set, MBPOLYGON, gather_set_cells );CHECK_ERR( rval );
475  CHECK_EQUAL( (size_t)642, gather_set_cells.size() );
476  CHECK_EQUAL( (size_t)1, gather_set_cells.psize() );
477 
478  // Check ke0 tag values on 4 gather set cells: first pentagon, last pentagon,
479  // first hexagon and last hexagon
480  EntityHandle cell_ents[] = { gather_set_cells[0], gather_set_cells[11], gather_set_cells[12],
481  gather_set_cells[641] };
482  double ke0_val[4];
483  rval = mb.tag_get_data( ke_tag0, &cell_ents[0], 4, ke0_val );CHECK_ERR( rval );
484 
485  CHECK_REAL_EQUAL( 15.001, ke0_val[0], eps );
486  CHECK_REAL_EQUAL( 15.012, ke0_val[1], eps );
487  CHECK_REAL_EQUAL( 16.013, ke0_val[2], eps );
488  CHECK_REAL_EQUAL( 16.642, ke0_val[3], eps );
489  }
490 }
491 
492 void multiple_loads_of_same_file( bool no_mixed_elements )
493 {
494  Core moab;
495  Interface& mb = moab;
496 
497  // Need a file set for nomesh to work right
498  EntityHandle file_set;
499  ErrorCode rval;
500  rval = mb.create_meshset( MESHSET_SET, file_set );CHECK_ERR( rval );
501 
502  // Read first only header information, no mesh, no variable
503  read_options = "PARALLEL=READ_PART;PARTITION;NOMESH;VARIABLE=;PARTITION_METHOD=TRIVIAL";
504  if( no_mixed_elements ) read_options += ";NO_MIXED_ELEMENTS";
505 
506  rval = mb.load_file( example.c_str(), &file_set, read_options.c_str() );CHECK_ERR( rval );
507 
508  // Create mesh, no variable
509  read_options = "PARALLEL=READ_PART;PARTITION;PARALLEL_RESOLVE_SHARED_ENTS;PARTITION_METHOD="
510  "TRIVIAL;VARIABLE=";
511  if( no_mixed_elements ) read_options += ";NO_MIXED_ELEMENTS";
512 
513  rval = mb.load_file( example.c_str(), &file_set, read_options.c_str() );CHECK_ERR( rval );
514 
515  // Read variable ke at timestep 0, no mesh
516  read_options = "PARALLEL=READ_PART;PARTITION;PARTITION_METHOD=TRIVIAL;NOMESH;VARIABLE=ke;TIMESTEP=0";
517  if( no_mixed_elements ) read_options += ";NO_MIXED_ELEMENTS";
518 
519  rval = mb.load_file( example.c_str(), &file_set, read_options.c_str() );CHECK_ERR( rval );
520 
521  Range local_verts;
522  rval = mb.get_entities_by_type( file_set, MBVERTEX, local_verts );CHECK_ERR( rval );
523 
524  Range local_edges;
525  rval = mb.get_entities_by_type( file_set, MBEDGE, local_edges );CHECK_ERR( rval );
526 
527  Range local_cells;
528  rval = mb.get_entities_by_type( file_set, MBPOLYGON, local_cells );CHECK_ERR( rval );
529 
530  ParallelComm* pcomm = ParallelComm::get_pcomm( &mb, 0 );
531  int procs = pcomm->proc_config().proc_size();
532  int rank = pcomm->proc_config().proc_rank();
533 
534  // Make check runs this test on two processors
535  if( 2 == procs )
536  {
537  CHECK_EQUAL( (size_t)321, local_cells.size() );
538 
539  // Check tag for cell variable ke at timestep 0
540  Tag ke_tag0;
541  rval = mb.tag_get_handle( "ke0", 1, MB_TYPE_DOUBLE, ke_tag0 );CHECK_ERR( rval );
542 
543  // Get ke0 tag values on 3 local cells
544  EntityHandle cell_ents[] = { local_cells[0], local_cells[160], local_cells[320] };
545  double ke0_val[3];
546  rval = mb.tag_get_data( ke_tag0, cell_ents, 3, ke0_val );CHECK_ERR( rval );
547 
548  if( 0 == rank )
549  {
550  CHECK_EQUAL( (size_t)1120, local_verts.size() );
551  CHECK_EQUAL( (size_t)1438, local_edges.size() );
552  CHECK_EQUAL( (size_t)1, local_cells.psize() );
553  CHECK_REAL_EQUAL( 15.001, ke0_val[0], eps );
554  CHECK_REAL_EQUAL( 16.161, ke0_val[1], eps );
555  CHECK_REAL_EQUAL( 16.321, ke0_val[2], eps );
556  }
557  else if( 1 == rank )
558  {
559  CHECK_EQUAL( (size_t)1122, local_verts.size() );
560  CHECK_EQUAL( (size_t)1444, local_edges.size() );
561  CHECK_EQUAL( (size_t)1, local_cells.psize() );
562 
563  CHECK_REAL_EQUAL( 16.322, ke0_val[0], eps );
564  CHECK_REAL_EQUAL( 16.482, ke0_val[1], eps );
565  CHECK_REAL_EQUAL( 16.642, ke0_val[2], eps );
566  }
567  }
568 }