ReadHDF5.cpp

Go to the documentation of this file.

/**
 * MOAB, a Mesh-Oriented datABase, is a software component for creating,
 * storing and accessing finite element mesh data.
 *
 * Copyright 2004 Sandia Corporation. Under the terms of Contract
 * DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government
 * retains certain rights in this software.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 */
 
//-------------------------------------------------------------------------
// Filename : ReadHDF5.cpp
//
// Purpose : HDF5 Writer
//
// Creator : Jason Kraftcheck
//
// Creation Date : 04/18/04
//-------------------------------------------------------------------------
 
#include <cassert>
#include "moab/MOABConfig.h"
/* Include our MPI header before any HDF5 because otherwise
 it will get included indirectly by HDF5 */
#ifdef MOAB_HAVE_MPI
#include "moab_mpi.h"
#include "moab/ParallelComm.hpp"
#endif
#include <H5Tpublic.h>
#include <H5Ppublic.h>
#include <H5Epublic.h>
#include "moab/Interface.hpp"
#include "Internals.hpp"
#include "MBTagConventions.hpp"
#include "ReadHDF5.hpp"
#include "moab/CN.hpp"
#include "moab/FileOptions.hpp"
#include "moab/CpuTimer.hpp"
#ifdef MOAB_HAVE_HDF5_PARALLEL
#include <H5FDmpi.h>
#include <H5FDmpio.h>
#endif
//#include "WriteHDF5.hpp"
 
#include <cstdlib>
#include <cstring>
#include <limits>
#include <functional>
#include <iostream>
 
#include "IODebugTrack.hpp"
#include "ReadHDF5Dataset.hpp"
#include "ReadHDF5VarLen.hpp"
#include "moab_mpe.h"
 
namespace moab
{
 
/* If true, coordinates are read in blocked format (all X values before
 * Y values before Z values.) If undefined, then all coordinates for a
 * given vertex are read at the same time.
 */
const bool DEFAULT_BLOCKED_COORDINATE_IO = false;
 
/* If true, file is opened first by root node only to read summary,
 * file is the closed and the summary is broadcast to all nodes, after
 * which all nodes open file in parallel to read data. If undefined,
 * file is opened once in parallel and all nodes read summary data.
 */
const bool DEFAULT_BCAST_SUMMARY = true;
 
/* If true and all processors are to read the same block of data,
 * read it on one and broadcast to others rather than using collective
 * io
 */
const bool DEFAULT_BCAST_DUPLICATE_READS = true;
 
#define READ_HDF5_BUFFER_SIZE ( 128 * 1024 * 1024 )
 
#define assert_range( PTR, CNT ) \
 assert( ( PTR ) >= (void*)dataBuffer ); \
 assert( ( ( PTR ) + ( CNT ) ) <= (void*)( dataBuffer + bufferSize ) );
 
// Call \c error function during HDF5 library errors to make
// it easier to trap such errors in the debugger. This function
// gets registered with the HDF5 library as a callback. It
// works the same as the default (H5Eprint), except that it
// also calls the \c error function as a no-op.
#if defined( H5E_auto_t_vers ) && H5E_auto_t_vers > 1
static herr_t handle_hdf5_error( hid_t stack, void* data )
{
 ReadHDF5::HDF5ErrorHandler* h = reinterpret_cast< ReadHDF5::HDF5ErrorHandler* >( data );
 herr_t result = 0;
 if( h->func ) result = ( *h->func )( stack, h->data );MB_CHK_ERR_CONT( MB_FAILURE );
 return result;
}
#else
static herr_t handle_hdf5_error( void* data )
{
 ReadHDF5::HDF5ErrorHandler* h = reinterpret_cast< ReadHDF5::HDF5ErrorHandler* >( data );
 herr_t result = 0;
 if( h->func ) result = ( *h->func )( h->data );MB_CHK_ERR_CONT( MB_FAILURE );
 return result;
}
#endif
 
static void copy_sorted_file_ids( const EntityHandle* sorted_ids, long num_ids, Range& results )
{
 Range::iterator hint = results.begin();
 long i = 0;
 while( i < num_ids )
 {
 EntityHandle start = sorted_ids[i];
 for( ++i; i < num_ids && sorted_ids[i] == 1 + sorted_ids[i - 1]; ++i )
 ;
 hint = results.insert( hint, start, sorted_ids[i - 1] );
 }
}
 
static void intersect( const mhdf_EntDesc& group, const Range& range, Range& result )
{
 Range::const_iterator s, e;
 s = Range::lower_bound( range.begin(), range.end(), group.start_id );
 e = Range::lower_bound( s, range.end(), group.start_id + group.count );
 result.merge( s, e );
}
 
#define debug_barrier() debug_barrier_line( __LINE__ )
void ReadHDF5::debug_barrier_line( int lineno )
{
#ifdef MOAB_HAVE_MPI
 if( mpiComm )
 {
 const unsigned threshold = 2;
 static unsigned long count = 0;
 if( dbgOut.get_verbosity() >= threshold )
 {
 dbgOut.printf( threshold, "*********** Debug Barrier %lu (@%d)***********\n", ++count, lineno );
 MPI_Barrier( *mpiComm );
 }
 }
#else
 if( lineno )
 {
 }
#endif
}
 
class CheckOpenReadHDF5Handles
{
 int fileline;
 mhdf_FileHandle handle;
 int enter_count;
 
 public:
 CheckOpenReadHDF5Handles( mhdf_FileHandle file, int line )
 : fileline( line ), handle( file ), enter_count( mhdf_countOpenHandles( file ) )
 {
 }
 ~CheckOpenReadHDF5Handles()
 {
 int new_count = mhdf_countOpenHandles( handle );
 if( new_count != enter_count )
 {
 std::cout << "Leaked HDF5 object handle in function at " << __FILE__ << ":" << fileline << std::endl
 << "Open at entrance: " << enter_count << std::endl
 << "Open at exit: " << new_count << std::endl;
 }
 }
};
 
#ifdef NDEBUG
#define CHECK_OPEN_HANDLES
#else
#define CHECK_OPEN_HANDLES CheckOpenReadHDF5Handles check_open_handles_( filePtr, __LINE__ )
#endif
 
ReaderIface* ReadHDF5::factory( Interface* iface )
{
 return new ReadHDF5( iface );
}
 
ReadHDF5::ReadHDF5( Interface* iface )
 : bufferSize( READ_HDF5_BUFFER_SIZE ), dataBuffer( NULL ), iFace( iface ), filePtr( 0 ), fileInfo( NULL ),
 readUtil( NULL ), handleType( 0 ), indepIO( H5P_DEFAULT ), collIO( H5P_DEFAULT ), myPcomm( NULL ),
 debugTrack( false ), dbgOut( stderr ), nativeParallel( false ), mpiComm( NULL ),
 blockedCoordinateIO( DEFAULT_BLOCKED_COORDINATE_IO ), bcastSummary( DEFAULT_BCAST_SUMMARY ),
 bcastDuplicateReads( DEFAULT_BCAST_DUPLICATE_READS ), setMeta( 0 ), timer( NULL ), cputime( false )
{
}
 
ErrorCode ReadHDF5::init()
{
 ErrorCode rval;
 
 if( readUtil ) return MB_SUCCESS;
 
 indepIO = collIO = H5P_DEFAULT;
 // WriteHDF5::register_known_tag_types(iFace);
 
 handleType = H5Tcopy( H5T_NATIVE_ULONG );
 if( handleType < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 if( H5Tset_size( handleType, sizeof( EntityHandle ) ) < 0 )
 {
 H5Tclose( handleType );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 rval = iFace->query_interface( readUtil );
 if( MB_SUCCESS != rval )
 {
 H5Tclose( handleType );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 idMap.clear();
 fileInfo = 0;
 debugTrack = false;
 myPcomm = 0;
 
 return MB_SUCCESS;
}
 
ReadHDF5::~ReadHDF5()
{
 if( !readUtil ) // init() failed.
 return;
 
 delete[] setMeta;
 setMeta = 0;
 iFace->release_interface( readUtil );
 H5Tclose( handleType );
}
 
ErrorCode ReadHDF5::set_up_read( const char* filename, const FileOptions& opts )
{
 ErrorCode rval;
 mhdf_Status status;
 indepIO = collIO = H5P_DEFAULT;
 mpiComm = 0;
 
 if( MB_SUCCESS != init() ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
 herr_t err = H5Eget_auto( H5E_DEFAULT, &errorHandler.func, &errorHandler.data );
#else
 herr_t err = H5Eget_auto( &errorHandler.func, &errorHandler.data );
#endif
 if( err < 0 )
 {
 errorHandler.func = 0;
 errorHandler.data = 0;
 }
 else
 {
#if defined( H5Eset_auto_vers ) && H5Eset_auto_vers > 1
 err = H5Eset_auto( H5E_DEFAULT, &handle_hdf5_error, &errorHandler );
#else
 err = H5Eset_auto( &handle_hdf5_error, &errorHandler );
#endif
 if( err < 0 )
 {
 errorHandler.func = 0;
 errorHandler.data = 0;
 }
 }
 
 // Set up debug output
 int tmpval;
 if( MB_SUCCESS == opts.get_int_option( "DEBUG_IO", 1, tmpval ) )
 {
 dbgOut.set_verbosity( tmpval );
 dbgOut.set_prefix( "H5M " );
 }
 dbgOut.limit_output_to_first_N_procs( 32 );
 
 // Enable some extra checks for reads. Note: amongst other things this
 // will print errors if the entire file is not read, so if doing a
 // partial read that is not a parallel read, this should be disabled.
 debugTrack = ( MB_SUCCESS == opts.get_null_option( "DEBUG_BINIO" ) );
 
 opts.get_toggle_option( "BLOCKED_COORDINATE_IO", DEFAULT_BLOCKED_COORDINATE_IO, blockedCoordinateIO );
 opts.get_toggle_option( "BCAST_SUMMARY", DEFAULT_BCAST_SUMMARY, bcastSummary );
 opts.get_toggle_option( "BCAST_DUPLICATE_READS", DEFAULT_BCAST_DUPLICATE_READS, bcastDuplicateReads );
 
 // Handle parallel options
 bool use_mpio = ( MB_SUCCESS == opts.get_null_option( "USE_MPIO" ) );
 rval = opts.match_option( "PARALLEL", "READ_PART" );
 bool parallel = ( rval != MB_ENTITY_NOT_FOUND );
 nativeParallel = ( rval == MB_SUCCESS );
 if( use_mpio && !parallel )
 {
 MB_SET_ERR( MB_NOT_IMPLEMENTED, "'USE_MPIO' option specified w/out 'PARALLEL' option" );
 }
 
 // This option is intended for testing purposes only, and thus
 // is not documented anywhere. Decreasing the buffer size can
 // expose bugs that would otherwise only be seen when reading
 // very large files.
 rval = opts.get_int_option( "BUFFER_SIZE", bufferSize );
 if( MB_SUCCESS != rval )
 {
 bufferSize = READ_HDF5_BUFFER_SIZE;
 }
 else if( bufferSize < (int)std::max( sizeof( EntityHandle ), sizeof( void* ) ) )
 {
 MB_CHK_ERR( MB_INVALID_SIZE );
 }
 
 dataBuffer = (char*)malloc( bufferSize );
 if( !dataBuffer ) MB_CHK_ERR( MB_MEMORY_ALLOCATION_FAILED );
 
 if( use_mpio || nativeParallel )
 {
 
#ifndef MOAB_HAVE_HDF5_PARALLEL
 free( dataBuffer );
 dataBuffer = NULL;
 MB_SET_ERR( MB_NOT_IMPLEMENTED, "MOAB not configured with parallel HDF5 support" );
#else
 MPI_Info info = MPI_INFO_NULL;
 std::string cb_size;
 rval = opts.get_str_option( "CB_BUFFER_SIZE", cb_size );
 if( MB_SUCCESS == rval )
 {
 MPI_Info_create( &info );
 MPI_Info_set( info, const_cast< char* >( "cb_buffer_size" ), const_cast< char* >( cb_size.c_str() ) );
 }
 
 int pcomm_no = 0;
 rval = opts.get_int_option( "PARALLEL_COMM", pcomm_no );
 if( rval == MB_TYPE_OUT_OF_RANGE )
 {
 MB_SET_ERR( rval, "Invalid value for PARALLEL_COMM option" );
 }
 myPcomm = ParallelComm::get_pcomm( iFace, pcomm_no );
 if( 0 == myPcomm )
 {
 myPcomm = new ParallelComm( iFace, MPI_COMM_WORLD );
 }
 const int rank = myPcomm->proc_config().proc_rank();
 dbgOut.set_rank( rank );
 dbgOut.limit_output_to_first_N_procs( 32 );
 mpiComm = new MPI_Comm( myPcomm->proc_config().proc_comm() );
 
#ifndef H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS
 dbgOut.print( 1, "H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS is not defined\n" );
#endif
 
 // Open the file in serial on root to read summary
 dbgOut.tprint( 1, "Getting file summary\n" );
 fileInfo = 0;
 
 hid_t file_prop;
 if( bcastSummary )
 {
 unsigned long size = 0;
 if( rank == 0 )
 {
 file_prop = H5Pcreate( H5P_FILE_ACCESS );
 err = H5Pset_fapl_mpio( file_prop, MPI_COMM_SELF, MPI_INFO_NULL );
 assert( file_prop >= 0 );
 assert( err >= 0 );
 filePtr = mhdf_openFileWithOpt( filename, 0, NULL, handleType, file_prop, &status );
 H5Pclose( file_prop );
 
 if( filePtr )
 {
 fileInfo = mhdf_getFileSummary( filePtr, handleType, &status,
 0 ); // no extra set info
 if( !is_error( status ) )
 {
 size = fileInfo->total_size;
 fileInfo->offset = (unsigned char*)fileInfo;
 }
 }
 mhdf_closeFile( filePtr, &status );
 if( fileInfo && mhdf_isError( &status ) )
 {
 free( fileInfo );
 fileInfo = NULL;
 }
 }
 
 dbgOut.tprint( 1, "Communicating file summary\n" );
 int mpi_err = MPI_Bcast( &size, 1, MPI_UNSIGNED_LONG, 0, myPcomm->proc_config().proc_comm() );
 if( mpi_err || !size ) return MB_FAILURE;
 
 if( rank != 0 ) fileInfo = reinterpret_cast< mhdf_FileDesc* >( malloc( size ) );
 
 MPI_Bcast( fileInfo, size, MPI_BYTE, 0, myPcomm->proc_config().proc_comm() );
 
 if( rank != 0 ) mhdf_fixFileDesc( fileInfo, reinterpret_cast< mhdf_FileDesc* >( fileInfo->offset ) );
 }
 
 file_prop = H5Pcreate( H5P_FILE_ACCESS );
 err = H5Pset_fapl_mpio( file_prop, myPcomm->proc_config().proc_comm(), info );
 assert( file_prop >= 0 );
 assert( err >= 0 );
 
 collIO = H5Pcreate( H5P_DATASET_XFER );
 assert( collIO > 0 );
 err = H5Pset_dxpl_mpio( collIO, H5FD_MPIO_COLLECTIVE );
 assert( err >= 0 );
 indepIO = nativeParallel ? H5P_DEFAULT : collIO;
 
 // Re-open file in parallel
 dbgOut.tprintf( 1, "Opening \"%s\" for parallel IO\n", filename );
 filePtr = mhdf_openFileWithOpt( filename, 0, NULL, handleType, file_prop, &status );
 
 H5Pclose( file_prop );
 if( !filePtr )
 {
 free( dataBuffer );
 dataBuffer = NULL;
 H5Pclose( indepIO );
 if( collIO != indepIO ) H5Pclose( collIO );
 collIO = indepIO = H5P_DEFAULT;
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 if( !bcastSummary )
 {
 fileInfo = mhdf_getFileSummary( filePtr, handleType, &status, 0 );
 if( is_error( status ) )
 {
 free( dataBuffer );
 dataBuffer = NULL;
 mhdf_closeFile( filePtr, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
#endif // HDF5_PARALLEL
 }
 else
 {
 // Open the file
 filePtr = mhdf_openFile( filename, 0, NULL, handleType, &status );
 if( !filePtr )
 {
 free( dataBuffer );
 dataBuffer = NULL;
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 // Get file info
 fileInfo = mhdf_getFileSummary( filePtr, handleType, &status, 0 );
 if( is_error( status ) )
 {
 free( dataBuffer );
 dataBuffer = NULL;
 mhdf_closeFile( filePtr, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 
 ReadHDF5Dataset::default_hyperslab_selection_limit();
 int hslimit;
 rval = opts.get_int_option( "HYPERSLAB_SELECT_LIMIT", hslimit );
 if( MB_SUCCESS == rval && hslimit > 0 )
 ReadHDF5Dataset::set_hyperslab_selection_limit( hslimit );
 else
 ReadHDF5Dataset::default_hyperslab_selection_limit();
 if( MB_SUCCESS != opts.get_null_option( "HYPERSLAB_OR" ) &&
 ( MB_SUCCESS == opts.get_null_option( "HYPERSLAB_APPEND" ) || HDF5_can_append_hyperslabs() ) )
 {
 ReadHDF5Dataset::append_hyperslabs();
 if( MB_SUCCESS != opts.get_int_option( "HYPERSLAB_SELECT_LIMIT", hslimit ) )
 ReadHDF5Dataset::set_hyperslab_selection_limit( std::numeric_limits< int >::max() );
 dbgOut.print( 1, "Using H5S_APPEND for hyperslab selection\n" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::clean_up_read( const FileOptions& )
{
 HDF5ErrorHandler handler;
#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
 herr_t err = H5Eget_auto( H5E_DEFAULT, &handler.func, &handler.data );
#else
 herr_t err = H5Eget_auto( &handler.func, &handler.data );
#endif
 if( err >= 0 && handler.func == &handle_hdf5_error )
 {
 assert( handler.data == &errorHandler );
#if defined( H5Eget_auto_vers ) && H5Eget_auto_vers > 1
 H5Eset_auto( H5E_DEFAULT, errorHandler.func, errorHandler.data );
#else
 H5Eset_auto( errorHandler.func, errorHandler.data );
#endif
 }
 
 free( dataBuffer );
 dataBuffer = NULL;
 free( fileInfo );
 fileInfo = NULL;
 delete mpiComm;
 mpiComm = 0;
 
 if( indepIO != H5P_DEFAULT ) H5Pclose( indepIO );
 if( collIO != indepIO ) H5Pclose( collIO );
 collIO = indepIO = H5P_DEFAULT;
 
 delete[] setMeta;
 setMeta = 0;
 
 mhdf_Status status;
 mhdf_closeFile( filePtr, &status );
 filePtr = 0;
 return is_error( status ) ? MB_FAILURE : MB_SUCCESS;
}
 
ErrorCode ReadHDF5::load_file( const char* filename,
 const EntityHandle* file_set,
 const FileOptions& opts,
 const ReaderIface::SubsetList* subset_list,
 const Tag* file_id_tag )
{
 ErrorCode rval;
 
 rval = set_up_read( filename, opts );
 if( MB_SUCCESS != rval )
 {
 clean_up_read( opts );
 return rval;
 }
 // See if we need to report times
 
 rval = opts.get_null_option( "CPUTIME" );
 if( MB_SUCCESS == rval )
 {
 cputime = true;
 timer = new CpuTimer;
 for( int i = 0; i < NUM_TIMES; i++ )
 _times[i] = 0;
 }
 
 // We read the entire set description table regardless of partial
 // or complete reads or serial vs parallel reads
 rval = read_all_set_meta();
 
 if( cputime ) _times[SET_META_TIME] = timer->time_elapsed();
 if( subset_list && MB_SUCCESS == rval )
 rval = load_file_partial( subset_list->tag_list, subset_list->tag_list_length, subset_list->num_parts,
 subset_list->part_number, opts );
 else
 rval = load_file_impl( opts );
 
 if( MB_SUCCESS == rval && file_id_tag )
 {
 dbgOut.tprint( 1, "Storing file IDs in tag\n" );
 rval = store_file_ids( *file_id_tag );
 }
 ErrorCode rval3 = opts.get_null_option( "STORE_SETS_FILEIDS" );
 if( MB_SUCCESS == rval3 )
 {
 rval = store_sets_file_ids();
 if( MB_SUCCESS != rval ) return rval;
 }
 
 if( cputime ) _times[STORE_FILE_IDS_TIME] = timer->time_elapsed();
 
 if( MB_SUCCESS == rval && 0 != file_set )
 {
 dbgOut.tprint( 1, "Reading QA records\n" );
 rval = read_qa( *file_set );
 }
 
 if( cputime ) _times[READ_QA_TIME] = timer->time_elapsed();
 dbgOut.tprint( 1, "Cleaning up\n" );
 ErrorCode rval2 = clean_up_read( opts );
 if( rval == MB_SUCCESS && rval2 != MB_SUCCESS ) rval = rval2;
 
 if( MB_SUCCESS == rval )
 dbgOut.tprint( 1, "Read finished.\n" );
 else
 {
 std::string msg;
 iFace->get_last_error( msg );
 dbgOut.tprintf( 1, "READ FAILED (ERROR CODE %s): %s\n", ErrorCodeStr[rval], msg.c_str() );
 }
 
 if( cputime )
 {
 _times[TOTAL_TIME] = timer->time_since_birth();
 print_times();
 delete timer;
 }
 if( H5P_DEFAULT != collIO ) H5Pclose( collIO );
 if( H5P_DEFAULT != indepIO ) H5Pclose( indepIO );
 collIO = indepIO = H5P_DEFAULT;
 
 return rval;
}
 
ErrorCode ReadHDF5::load_file_impl( const FileOptions& )
{
 ErrorCode rval;
 mhdf_Status status;
 int i;
 
 CHECK_OPEN_HANDLES;
 
 dbgOut.tprint( 1, "Reading all nodes...\n" );
 Range ids;
 if( fileInfo->nodes.count )
 {
 ids.insert( fileInfo->nodes.start_id, fileInfo->nodes.start_id + fileInfo->nodes.count - 1 );
 rval = read_nodes( ids );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 dbgOut.tprint( 1, "Reading all element connectivity...\n" );
 std::vector< int > polyhedra; // Need to do these last so that faces are loaded
 for( i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 if( CN::EntityTypeFromName( fileInfo->elems[i].type ) == MBPOLYHEDRON )
 {
 polyhedra.push_back( i );
 continue;
 }
 
 rval = read_elems( i );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 for( std::vector< int >::iterator it = polyhedra.begin(); it != polyhedra.end(); ++it )
 {
 rval = read_elems( *it );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 dbgOut.tprint( 1, "Reading all sets...\n" );
 ids.clear();
 if( fileInfo->sets.count )
 {
 ids.insert( fileInfo->sets.start_id, fileInfo->sets.start_id + fileInfo->sets.count - 1 );
 rval = read_sets( ids );
 if( rval != MB_SUCCESS )
 {
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 dbgOut.tprint( 1, "Reading all adjacencies...\n" );
 for( i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 if( !fileInfo->elems[i].have_adj ) continue;
 
 long table_len;
 hid_t table = mhdf_openAdjacency( filePtr, fileInfo->elems[i].handle, &table_len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 rval = read_adjacencies( table, table_len );
 mhdf_closeData( filePtr, table, &status );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 dbgOut.tprint( 1, "Reading all tags...\n" );
 for( i = 0; i < fileInfo->num_tag_desc; ++i )
 {
 rval = read_tag( i );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 dbgOut.tprint( 1, "Core read finished. Cleaning up...\n" );
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::find_int_tag( const char* name, int& index )
{
 for( index = 0; index < fileInfo->num_tag_desc; ++index )
 if( !strcmp( name, fileInfo->tags[index].name ) ) break;
 
 if( index == fileInfo->num_tag_desc )
 {
 MB_SET_ERR( MB_TAG_NOT_FOUND, "File does not contain subset tag '" << name << "'" );
 }
 
 if( fileInfo->tags[index].type != mhdf_INTEGER || fileInfo->tags[index].size != 1 )
 {
 MB_SET_ERR( MB_TAG_NOT_FOUND, "Tag ' " << name << "' does not contain single integer value" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::get_subset_ids( const ReaderIface::IDTag* subset_list, int subset_list_length, Range& file_ids )
{
 ErrorCode rval;
 
 for( int i = 0; i < subset_list_length; ++i )
 {
 int tag_index;
 rval = find_int_tag( subset_list[i].tag_name, tag_index );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 Range tmp_file_ids;
 if( !subset_list[i].num_tag_values )
 {
 rval = get_tagged_entities( tag_index, tmp_file_ids );
 }
 else
 {
 std::vector< int > ids( subset_list[i].tag_values,
 subset_list[i].tag_values + subset_list[i].num_tag_values );
 std::sort( ids.begin(), ids.end() );
 rval = search_tag_values( tag_index, ids, tmp_file_ids );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( tmp_file_ids.empty() ) MB_CHK_ERR( MB_ENTITY_NOT_FOUND );
 
 if( i == 0 )
 file_ids.swap( tmp_file_ids );
 else
 file_ids = intersect( tmp_file_ids, file_ids );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::get_partition( Range& tmp_file_ids, int num_parts, int part_number )
{
 CHECK_OPEN_HANDLES;
 
 // Check that the tag only identified sets
 if( (unsigned long)fileInfo->sets.start_id > tmp_file_ids.front() )
 {
 dbgOut.print( 2, "Ignoring non-set entities with partition set tag\n" );
 tmp_file_ids.erase( tmp_file_ids.begin(), tmp_file_ids.lower_bound( (EntityHandle)fileInfo->sets.start_id ) );
 }
 unsigned long set_end = (unsigned long)fileInfo->sets.start_id + fileInfo->sets.count;
 if( tmp_file_ids.back() >= set_end )
 {
 dbgOut.print( 2, "Ignoring non-set entities with partition set tag\n" );
 tmp_file_ids.erase( tmp_file_ids.upper_bound( (EntityHandle)set_end ), tmp_file_ids.end() );
 }
 
 Range::iterator s = tmp_file_ids.begin();
 size_t num_per_proc = tmp_file_ids.size() / num_parts;
 size_t num_extra = tmp_file_ids.size() % num_parts;
 Range::iterator e;
 if( part_number < (long)num_extra )
 {
 s += ( num_per_proc + 1 ) * part_number;
 e = s;
 e += ( num_per_proc + 1 );
 }
 else
 {
 s += num_per_proc * part_number + num_extra;
 e = s;
 e += num_per_proc;
 }
 tmp_file_ids.erase( e, tmp_file_ids.end() );
 tmp_file_ids.erase( tmp_file_ids.begin(), s );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::load_file_partial( const ReaderIface::IDTag* subset_list,
 int subset_list_length,
 int num_parts,
 int part_number,
 const FileOptions& opts )
{
 mhdf_Status status;
 
 static MPEState mpe_event( "ReadHDF5", "yellow" );
 
 mpe_event.start( "gather parts" );
 
 CHECK_OPEN_HANDLES;
 
 for( int i = 0; i < subset_list_length; ++i )
 {
 dbgOut.printf( 2, "Select by \"%s\" with num_tag_values = %d\n", subset_list[i].tag_name,
 subset_list[i].num_tag_values );
 if( subset_list[i].num_tag_values )
 {
 assert( 0 != subset_list[i].tag_values );
 dbgOut.printf( 2, " \"%s\" values = { %d", subset_list[i].tag_name, subset_list[i].tag_values[0] );
 for( int j = 1; j < subset_list[i].num_tag_values; ++j )
 dbgOut.printf( 2, ", %d", subset_list[i].tag_values[j] );
 dbgOut.printf( 2, " }\n" );
 }
 }
 if( num_parts ) dbgOut.printf( 2, "Partition with num_parts = %d and part_number = %d\n", num_parts, part_number );
 
 dbgOut.tprint( 1, "RETRIEVING TAGGED ENTITIES\n" );
 
 Range file_ids;
 ErrorCode rval = get_subset_ids( subset_list, subset_list_length, file_ids );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[SUBSET_IDS_TIME] = timer->time_elapsed();
 
 if( num_parts )
 {
 /*if (num_parts>(int)file_ids.size())
 {
 MB_SET_ERR(MB_FAILURE, "Only " << file_ids.size() << " parts to distribute to " <<
 num_parts << " processes.");
 }*/
 rval = get_partition( file_ids, num_parts, part_number );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[GET_PARTITION_TIME] = timer->time_elapsed();
 
 dbgOut.print_ints( 4, "Set file IDs for partial read: ", file_ids );
 mpe_event.end();
 mpe_event.start( "gather related sets" );
 dbgOut.tprint( 1, "GATHERING ADDITIONAL ENTITIES\n" );
 
 enum RecusiveSetMode
 {
 RSM_NONE,
 RSM_SETS,
 RSM_CONTENTS
 };
 const char* const set_opts[] = { "NONE", "SETS", "CONTENTS", NULL };
 int child_mode;
 rval = opts.match_option( "CHILDREN", set_opts, child_mode );
 if( MB_ENTITY_NOT_FOUND == rval )
 child_mode = RSM_CONTENTS;
 else if( MB_SUCCESS != rval )
 {
 MB_SET_ERR( rval, "Invalid value for 'CHILDREN' option" );
 }
 int content_mode;
 rval = opts.match_option( "SETS", set_opts, content_mode );
 if( MB_ENTITY_NOT_FOUND == rval )
 content_mode = RSM_CONTENTS;
 else if( MB_SUCCESS != rval )
 {
 MB_SET_ERR( rval, "Invalid value for 'SETS' option" );
 }
 
 // If we want the contents of contained/child sets,
 // search for them now (before gathering the non-set contents
 // of the sets.)
 Range sets;
 intersect( fileInfo->sets, file_ids, sets );
 if( content_mode == RSM_CONTENTS || child_mode == RSM_CONTENTS )
 {
 dbgOut.tprint( 1, " doing read_set_ids_recursive\n" );
 rval = read_set_ids_recursive( sets, content_mode == RSM_CONTENTS, child_mode == RSM_CONTENTS );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[GET_SET_IDS_TIME] = timer->time_elapsed();
 debug_barrier();
 
 // Get elements and vertices contained in sets
 dbgOut.tprint( 1, " doing get_set_contents\n" );
 rval = get_set_contents( sets, file_ids );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[GET_SET_CONTENTS_TIME] = timer->time_elapsed();
 
 dbgOut.print_ints( 5, "File IDs for partial read: ", file_ids );
 debug_barrier();
 mpe_event.end();
 dbgOut.tprint( 1, "GATHERING NODE IDS\n" );
 
 // Figure out the maximum dimension of entity to be read
 int max_dim = 0;
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 EntityType type = CN::EntityTypeFromName( fileInfo->elems[i].type );
 if( type <= MBVERTEX || type >= MBENTITYSET )
 {
 assert( false ); // For debug code die for unknown element types
 continue; // For release code, skip unknown element types
 }
 int dim = CN::Dimension( type );
 if( dim > max_dim )
 {
 Range subset;
 intersect( fileInfo->elems[i].desc, file_ids, subset );
 if( !subset.empty() ) max_dim = dim;
 }
 }
#ifdef MOAB_HAVE_MPI
 if( nativeParallel )
 {
 int send = max_dim;
 MPI_Allreduce( &send, &max_dim, 1, MPI_INT, MPI_MAX, *mpiComm );
 }
#endif
 
 // If input contained any polyhedra, then need to get faces
 // of the polyhedra before the next loop because we need to
 // read said faces in that loop.
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 EntityType type = CN::EntityTypeFromName( fileInfo->elems[i].type );
 if( type != MBPOLYHEDRON ) continue;
 
 debug_barrier();
 dbgOut.print( 2, " Getting polyhedra faces\n" );
 mpe_event.start( "reading connectivity for ", fileInfo->elems[i].handle );
 
 Range polyhedra;
 intersect( fileInfo->elems[i].desc, file_ids, polyhedra );
 rval = read_elems( i, polyhedra, &file_ids );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[GET_POLYHEDRA_TIME] = timer->time_elapsed();
 // Get node file ids for all elements
 Range nodes;
 intersect( fileInfo->nodes, file_ids, nodes );
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 EntityType type = CN::EntityTypeFromName( fileInfo->elems[i].type );
 if( type <= MBVERTEX || type >= MBENTITYSET )
 {
 assert( false ); // For debug code die for unknown element types
 continue; // For release code, skip unknown element types
 }
 if( MBPOLYHEDRON == type ) continue;
 
 debug_barrier();
 dbgOut.printf( 2, " Getting element node IDs for: %s\n", fileInfo->elems[i].handle );
 
 Range subset;
 intersect( fileInfo->elems[i].desc, file_ids, subset );
 mpe_event.start( "reading connectivity for ", fileInfo->elems[i].handle );
 
 // If dimension is max_dim, then we can create the elements now
 // so we don't have to read the table again later (connectivity
 // will be fixed up after nodes are created when update_connectivity())
 // is called. For elements of a smaller dimension, we just build
 // the node ID range now because a) we'll have to read the whole
 // connectivity table again later, and b) we don't want to worry
 // about accidentally creating multiple copies of the same element.
 if( CN::Dimension( type ) == max_dim )
 rval = read_elems( i, subset, &nodes );
 else
 rval = read_elems( i, subset, nodes );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 if( cputime ) _times[GET_ELEMENTS_TIME] = timer->time_elapsed();
 debug_barrier();
 mpe_event.start( "read coords" );
 dbgOut.tprintf( 1, "READING NODE COORDINATES (%lu nodes in %lu selects)\n", (unsigned long)nodes.size(),
 (unsigned long)nodes.psize() );
 
 // Read node coordinates and create vertices in MOAB
 // NOTE: This populates the RangeMap with node file ids,
 // which is expected by read_node_adj_elems.
 rval = read_nodes( nodes );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[GET_NODES_TIME] = timer->time_elapsed();
 
 debug_barrier();
 dbgOut.tprint( 1, "READING ELEMENTS\n" );
 
 // Decide if we need to read additional elements
 enum SideMode
 {
 SM_EXPLICIT,
 SM_NODES,
 SM_SIDES
 };
 int side_mode;
 const char* const options[] = { "EXPLICIT", "NODES", "SIDES", 0 };
 rval = opts.match_option( "ELEMENTS", options, side_mode );
 if( MB_ENTITY_NOT_FOUND == rval )
 {
 // If only nodes were specified, then default to "NODES", otherwise
 // default to "SIDES".
 if( 0 == max_dim )
 side_mode = SM_NODES;
 else
 side_mode = SM_SIDES;
 }
 else if( MB_SUCCESS != rval )
 {
 MB_SET_ERR( rval, "Invalid value for 'ELEMENTS' option" );
 }
 
 if( side_mode == SM_SIDES /*ELEMENTS=SIDES*/ && max_dim == 0 /*node-based*/ )
 {
 // Read elements until we find something. Once we find something,
 // read only elements of the same dimension. NOTE: loop termination
 // criterion changes on both sides (max_dim can be changed in loop
 // body).
 for( int dim = 3; dim >= max_dim; --dim )
 {
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 EntityType type = CN::EntityTypeFromName( fileInfo->elems[i].type );
 if( CN::Dimension( type ) == dim )
 {
 debug_barrier();
 dbgOut.tprintf( 2, " Reading node-adjacent elements for: %s\n", fileInfo->elems[i].handle );
 mpe_event.start( "reading connectivity for ", fileInfo->elems[i].handle );
 Range ents;
 rval = read_node_adj_elems( fileInfo->elems[i] );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 if( !ents.empty() ) max_dim = 3;
 }
 }
 }
 }
 
 if( cputime ) _times[GET_NODEADJ_TIME] = timer->time_elapsed();
 Range side_entities;
 if( side_mode != SM_EXPLICIT /*ELEMENTS=NODES || ELEMENTS=SIDES*/ )
 {
 if( 0 == max_dim ) max_dim = 4;
 // Now read any additional elements for which we've already read all
 // of the nodes.
 for( int dim = max_dim - 1; dim > 0; --dim )
 {
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 EntityType type = CN::EntityTypeFromName( fileInfo->elems[i].type );
 if( CN::Dimension( type ) == dim )
 {
 debug_barrier();
 dbgOut.tprintf( 2, " Reading node-adjacent elements for: %s\n", fileInfo->elems[i].handle );
 mpe_event.start( "reading connectivity for ", fileInfo->elems[i].handle );
 rval = read_node_adj_elems( fileInfo->elems[i], &side_entities );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 }
 }
 
 // We need to do this here for polyhedra to be handled correctly.
 // We have to wait until the faces are read in the above code block,
 // but need to create the connectivity before doing update_connectivity,
 // which might otherwise delete polyhedra faces.
 if( cputime ) _times[GET_SIDEELEM_TIME] = timer->time_elapsed();
 
 debug_barrier();
 dbgOut.tprint( 1, "UPDATING CONNECTIVITY ARRAYS FOR READ ELEMENTS\n" );
 mpe_event.start( "updating connectivity for elements read before vertices" );
 rval = update_connectivity();
 mpe_event.end();
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[UPDATECONN_TIME] = timer->time_elapsed();
 
 dbgOut.tprint( 1, "READING ADJACENCIES\n" );
 for( int i = 0; i < fileInfo->num_elem_desc; ++i )
 {
 if (fileInfo->elems[i].have_adj /*&&
 idMap.intersects(fileInfo->elems[i].desc.start_id, fileInfo->elems[i].desc.count) */)
 {
 mpe_event.start( "reading adjacencies for ", fileInfo->elems[i].handle );
 long len;
 hid_t th = mhdf_openAdjacency( filePtr, fileInfo->elems[i].handle, &len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 rval = read_adjacencies( th, len );
 mhdf_closeData( filePtr, th, &status );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 if( cputime ) _times[ADJACENCY_TIME] = timer->time_elapsed();
 
 // If doing ELEMENTS=SIDES then we need to delete any entities
 // that we read that aren't actually sides (e.g. an interior face
 // that connects two disjoint portions of the part). Both
 // update_connectivity and reading of any explicit adjacencies must
 // happen before this.
 if( side_mode == SM_SIDES )
 {
 debug_barrier();
 mpe_event.start( "cleaning up non-side lower-dim elements" );
 dbgOut.tprint( 1, "CHECKING FOR AND DELETING NON-SIDE ELEMENTS\n" );
 rval = delete_non_side_elements( side_entities );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[DELETE_NON_SIDEELEM_TIME] = timer->time_elapsed();
 
 debug_barrier();
 dbgOut.tprint( 1, "READING SETS\n" );
 
 // If reading contained/child sets but not their contents then find
 // them now. If we were also reading their contents we would
 // have found them already.
 if( content_mode == RSM_SETS || child_mode == RSM_SETS )
 {
 dbgOut.tprint( 1, " doing read_set_ids_recursive\n" );
 mpe_event.start( "finding recursively contained sets" );
 rval = read_set_ids_recursive( sets, content_mode == RSM_SETS, child_mode == RSM_SETS );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[READ_SET_IDS_RECURS_TIME] = timer->time_elapsed();
 
 dbgOut.tprint( 1, " doing find_sets_containing\n" );
 mpe_event.start( "finding sets containing any read entities" );
 
 // Decide whether to read set-containing parents
 bool read_set_containing_parents = true;
 std::string tmp_opt;
 rval = opts.get_option( "NO_SET_CONTAINING_PARENTS", tmp_opt );
 if( MB_SUCCESS == rval ) read_set_containing_parents = false;
 
 // Append file IDs of sets containing any of the nodes or elements
 // we've read up to this point.
 rval = find_sets_containing( sets, read_set_containing_parents );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[FIND_SETS_CONTAINING_TIME] = timer->time_elapsed();
 
 // Now actually read all set data and instantiate sets in MOAB.
 // Get any contained sets out of file_ids.
 mpe_event.start( "reading set contents/parents/children" );
 EntityHandle first_set = fileInfo->sets.start_id;
 sets.merge( file_ids.lower_bound( first_set ), file_ids.lower_bound( first_set + fileInfo->sets.count ) );
 dbgOut.tprint( 1, " doing read_sets\n" );
 rval = read_sets( sets );
 mpe_event.end( rval );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cputime ) _times[READ_SETS_TIME] = timer->time_elapsed();
 
 dbgOut.tprint( 1, "READING TAGS\n" );
 
 for( int i = 0; i < fileInfo->num_tag_desc; ++i )
 {
 mpe_event.start( "reading tag: ", fileInfo->tags[i].name );
 rval = read_tag( i );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( cputime ) _times[READ_TAGS_TIME] = timer->time_elapsed();
 
 dbgOut.tprint( 1, "PARTIAL READ COMPLETE.\n" );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::search_tag_values( int tag_index,
 const std::vector< int >& sorted_values,
 Range& file_ids,
 bool sets_only )
{
 ErrorCode rval;
 mhdf_Status status;
 std::vector< EntityHandle >::iterator iter;
 const mhdf_TagDesc& tag = fileInfo->tags[tag_index];
 long size;
 long start_id;
 
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 
 // Do dense data
 
 hid_t table;
 const char* name;
 std::vector< EntityHandle > indices;
 // These are probably in order of dimension, so iterate
 // in reverse order to make Range insertions more efficient.
 std::vector< int > grp_indices( tag.dense_elem_indices, tag.dense_elem_indices + tag.num_dense_indices );
 for( std::vector< int >::reverse_iterator i = grp_indices.rbegin(); i != grp_indices.rend(); ++i )
 {
 int idx = *i;
 if( idx == -2 )
 {
 name = mhdf_set_type_handle();
 start_id = fileInfo->sets.start_id;
 }
 else if( sets_only )
 {
 continue;
 }
 else if( idx == -1 )
 {
 name = mhdf_node_type_handle();
 start_id = fileInfo->nodes.start_id;
 }
 else
 {
 if( idx < 0 || idx >= fileInfo->num_elem_desc ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 name = fileInfo->elems[idx].handle;
 start_id = fileInfo->elems[idx].desc.start_id;
 }
 table = mhdf_openDenseTagData( filePtr, tag.name, name, &size, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 rval = search_tag_values( table, size, sorted_values, indices );
 mhdf_closeData( filePtr, table, &status );
 if( MB_SUCCESS != rval || is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 // Convert from table indices to file IDs and add to result list
 std::sort( indices.begin(), indices.end(), std::greater< EntityHandle >() );
 std::transform( indices.begin(), indices.end(), range_inserter( file_ids ),
 // std::bind1st(std::plus<long>(), start_id));
 std::bind( std::plus< long >(), start_id, std::placeholders::_1 ) );
 indices.clear();
 }
 
 if( !tag.have_sparse ) return MB_SUCCESS;
 
 // Do sparse data
 
 hid_t tables[2];
 long junk; // Redundant value for non-variable-length tags
 mhdf_openSparseTagData( filePtr, tag.name, &size, &junk, tables, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 rval = search_tag_values( tables[1], size, sorted_values, indices );
 mhdf_closeData( filePtr, tables[1], &status );
 if( MB_SUCCESS != rval || is_error( status ) )
 {
 mhdf_closeData( filePtr, tables[0], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 // Convert to ranges
 std::sort( indices.begin(), indices.end() );
 std::vector< EntityHandle > ranges;
 iter = indices.begin();
 while( iter != indices.end() )
 {
 ranges.push_back( *iter );
 EntityHandle last = *iter;
 for( ++iter; iter != indices.end() && ( last + 1 ) == *iter; ++iter, ++last )
 ;
 ranges.push_back( last );
 }
 // Read file ids
 iter = ranges.begin();
 unsigned long offset = 0;
 while( iter != ranges.end() )
 {
 long begin = *iter;
 ++iter;
 long end = *iter;
 ++iter;
 mhdf_readSparseTagEntitiesWithOpt( tables[0], begin, end - begin + 1, handleType, &indices[offset], indepIO,
 &status );
 if( is_error( status ) )
 {
 mhdf_closeData( filePtr, tables[0], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 offset += end - begin + 1;
 }
 mhdf_closeData( filePtr, tables[0], &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 assert( offset == indices.size() );
 std::sort( indices.begin(), indices.end() );
 
 if( sets_only )
 {
 iter = std::lower_bound( indices.begin(), indices.end(),
 (EntityHandle)( fileInfo->sets.start_id + fileInfo->sets.count ) );
 indices.erase( iter, indices.end() );
 iter = std::lower_bound( indices.begin(), indices.end(), fileInfo->sets.start_id );
 indices.erase( indices.begin(), iter );
 }
 copy_sorted_file_ids( &indices[0], indices.size(), file_ids );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::get_tagged_entities( int tag_index, Range& file_ids )
{
 const mhdf_TagDesc& tag = fileInfo->tags[tag_index];
 
 CHECK_OPEN_HANDLES;
 
 // Do dense data
 Range::iterator hint = file_ids.begin();
 for( int i = 0; i < tag.num_dense_indices; ++i )
 {
 int idx = tag.dense_elem_indices[i];
 mhdf_EntDesc* ents;
 if( idx == -2 )
 ents = &fileInfo->sets;
 else if( idx == -1 )
 ents = &fileInfo->nodes;
 else
 {
 if( idx < 0 || idx >= fileInfo->num_elem_desc ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 ents = &( fileInfo->elems[idx].desc );
 }
 
 EntityHandle h = (EntityHandle)ents->start_id;
 hint = file_ids.insert( hint, h, h + ents->count - 1 );
 }
 
 if( !tag.have_sparse ) return MB_SUCCESS;
 
 // Do sparse data
 
 mhdf_Status status;
 hid_t tables[2];
 long size, junk;
 mhdf_openSparseTagData( filePtr, tag.name, &size, &junk, tables, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 mhdf_closeData( filePtr, tables[1], &status );
 if( is_error( status ) )
 {
 mhdf_closeData( filePtr, tables[0], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 hid_t file_type = H5Dget_type( tables[0] );
 if( file_type < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 hint = file_ids.begin();
 EntityHandle* buffer = reinterpret_cast< EntityHandle* >( dataBuffer );
 const long buffer_size = bufferSize / std::max( sizeof( EntityHandle ), H5Tget_size( file_type ) );
 long remaining = size, offset = 0;
 while( remaining )
 {
 long count = std::min( buffer_size, remaining );
 assert_range( buffer, count );
 mhdf_readSparseTagEntitiesWithOpt( *tables, offset, count, file_type, buffer, collIO, &status );
 if( is_error( status ) )
 {
 H5Tclose( file_type );
 mhdf_closeData( filePtr, *tables, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 H5Tconvert( file_type, handleType, count, buffer, NULL, H5P_DEFAULT );
 
 std::sort( buffer, buffer + count );
 for( long i = 0; i < count; ++i )
 hint = file_ids.insert( hint, buffer[i], buffer[i] );
 
 remaining -= count;
 offset += count;
 }
 
 H5Tclose( file_type );
 mhdf_closeData( filePtr, *tables, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::search_tag_values( hid_t tag_table,
 unsigned long table_size,
 const std::vector< int >& sorted_values,
 std::vector< EntityHandle >& value_indices )
{
 debug_barrier();
 
 CHECK_OPEN_HANDLES;
 
 mhdf_Status status;
 size_t chunk_size = bufferSize / sizeof( int );
 int* buffer = reinterpret_cast< int* >( dataBuffer );
 size_t remaining = table_size, offset = 0;
 while( remaining )
 {
 // Get a block of tag values
 size_t count = std::min( chunk_size, remaining );
 assert_range( buffer, count );
 mhdf_readTagValuesWithOpt( tag_table, offset, count, H5T_NATIVE_INT, buffer, collIO, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 // Search tag values
 for( size_t i = 0; i < count; ++i )
 if( std::binary_search( sorted_values.begin(), sorted_values.end(), (int)buffer[i] ) )
 value_indices.push_back( i + offset );
 
 offset += count;
 remaining -= count;
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_nodes( const Range& node_file_ids )
{
 ErrorCode rval;
 mhdf_Status status;
 const int dim = fileInfo->nodes.vals_per_ent;
 Range range;
 
 CHECK_OPEN_HANDLES;
 
 if( node_file_ids.empty() && !nativeParallel ) return MB_SUCCESS;
 
 int cdim;
 rval = iFace->get_dimension( cdim );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( cdim < dim )
 {
 rval = iFace->set_dimension( dim );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 hid_t data_id = mhdf_openNodeCoordsSimple( filePtr, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 EntityHandle handle;
 std::vector< double* > arrays( dim );
 const size_t num_nodes = node_file_ids.size();
 if( num_nodes > 0 )
 {
 rval = readUtil->get_node_coords( dim, (int)num_nodes, 0, handle, arrays );
 if( MB_SUCCESS != rval )
 {
 mhdf_closeData( filePtr, data_id, &status );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 if( blockedCoordinateIO )
 {
 try
 {
 for( int d = 0; d < dim; ++d )
 {
 ReadHDF5Dataset reader( "blocked coords", data_id, nativeParallel, mpiComm, false );
 reader.set_column( d );
 reader.set_file_ids( node_file_ids, fileInfo->nodes.start_id, num_nodes, H5T_NATIVE_DOUBLE );
 dbgOut.printf( 3, "Reading %lu chunks for coordinate dimension %d\n", reader.get_read_count(), d );
 // Should normally only have one read call, unless sparse nature
 // of file_ids caused reader to do something strange
 size_t count, offset = 0;
 int nn = 0;
 while( !reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d for dimension %d\n", ++nn, d );
 reader.read( arrays[d] + offset, count );
 offset += count;
 }
 if( offset != num_nodes )
 {
 mhdf_closeData( filePtr, data_id, &status );
 assert( false );
 return MB_FAILURE;
 }
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 mhdf_closeData( filePtr, data_id, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 else
 { // !blockedCoordinateIO
 double* buffer = (double*)dataBuffer;
 long chunk_size = bufferSize / ( 3 * sizeof( double ) );
 long coffset = 0;
 int nn = 0;
 try
 {
 ReadHDF5Dataset reader( "interleaved coords", data_id, nativeParallel, mpiComm, false );
 reader.set_file_ids( node_file_ids, fileInfo->nodes.start_id, chunk_size, H5T_NATIVE_DOUBLE );
 dbgOut.printf( 3, "Reading %lu chunks for coordinate coordinates\n", reader.get_read_count() );
 while( !reader.done() )
 {
 dbgOut.tprintf( 3, "Reading chunk %d of node coords\n", ++nn );
 
 size_t count;
 reader.read( buffer, count );
 
 for( size_t i = 0; i < count; ++i )
 for( int d = 0; d < dim; ++d )
 arrays[d][coffset + i] = buffer[dim * i + d];
 coffset += count;
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 mhdf_closeData( filePtr, data_id, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 
 dbgOut.print( 3, "Closing node coordinate table\n" );
 mhdf_closeData( filePtr, data_id, &status );
 for( int d = dim; d < cdim; ++d )
 memset( arrays[d], 0, num_nodes * sizeof( double ) );
 
 dbgOut.printf( 3, "Updating ID to handle map for %lu nodes\n", (unsigned long)node_file_ids.size() );
 return insert_in_id_map( node_file_ids, handle );
}
 
ErrorCode ReadHDF5::read_elems( int i )
{
 Range ids;
 ids.insert( fileInfo->elems[i].desc.start_id,
 fileInfo->elems[i].desc.start_id + fileInfo->elems[i].desc.count - 1 );
 return read_elems( i, ids );
}
 
ErrorCode ReadHDF5::read_elems( int i, const Range& file_ids, Range* node_ids )
{
 if( fileInfo->elems[i].desc.vals_per_ent < 0 )
 {
 if( node_ids != 0 ) // Not implemented for version 3 format of poly data
 MB_CHK_ERR( MB_TYPE_OUT_OF_RANGE );
 return read_poly( fileInfo->elems[i], file_ids );
 }
 else
 return read_elems( fileInfo->elems[i], file_ids, node_ids );
}
 
ErrorCode ReadHDF5::read_elems( const mhdf_ElemDesc& elems, const Range& file_ids, Range* node_ids )
{
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 dbgOut.tprintf( 1, "READING %s CONNECTIVITY (%lu elems in %lu selects)\n", elems.handle,
 (unsigned long)file_ids.size(), (unsigned long)file_ids.psize() );
 
 ErrorCode rval = MB_SUCCESS;
 mhdf_Status status;
 
 EntityType type = CN::EntityTypeFromName( elems.type );
 if( type == MBMAXTYPE )
 {
 MB_SET_ERR( MB_FAILURE, "Unknown element type: \"" << elems.type << "\"" );
 }
 
 const int nodes_per_elem = elems.desc.vals_per_ent;
 const size_t count = file_ids.size();
 hid_t data_id = mhdf_openConnectivitySimple( filePtr, elems.handle, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 EntityHandle handle;
 EntityHandle* array = 0;
 if( count > 0 ) rval = readUtil->get_element_connect( count, nodes_per_elem, type, 0, handle, array );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 try
 {
 EntityHandle* buffer = reinterpret_cast< EntityHandle* >( dataBuffer );
 const size_t buffer_size = bufferSize / ( sizeof( EntityHandle ) * nodes_per_elem );
 ReadHDF5Dataset reader( elems.handle, data_id, nativeParallel, mpiComm );
 reader.set_file_ids( file_ids, elems.desc.start_id, buffer_size, handleType );
 dbgOut.printf( 3, "Reading connectivity in %lu chunks for element group \"%s\"\n", reader.get_read_count(),
 elems.handle );
 EntityHandle* iter = array;
 int nn = 0;
 while( !reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d for \"%s\"\n", ++nn, elems.handle );
 
 size_t num_read;
 reader.read( buffer, num_read );
 iter = std::copy( buffer, buffer + num_read * nodes_per_elem, iter );
 
 if( node_ids )
 {
 std::sort( buffer, buffer + num_read * nodes_per_elem );
 num_read = std::unique( buffer, buffer + num_read * nodes_per_elem ) - buffer;
 copy_sorted_file_ids( buffer, num_read, *node_ids );
 }
 }
 assert( iter - array == (ptrdiff_t)count * nodes_per_elem );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 if( !node_ids )
 {
 rval = convert_id_to_handle( array, count * nodes_per_elem );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 rval = readUtil->update_adjacencies( handle, count, nodes_per_elem, array );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 else
 {
 IDConnectivity t;
 t.handle = handle;
 t.count = count;
 t.nodes_per_elem = nodes_per_elem;
 t.array = array;
 idConnectivityList.push_back( t );
 }
 
 return insert_in_id_map( file_ids, handle );
}
 
ErrorCode ReadHDF5::update_connectivity()
{
 ErrorCode rval;
 std::vector< IDConnectivity >::iterator i;
 for( i = idConnectivityList.begin(); i != idConnectivityList.end(); ++i )
 {
 rval = convert_id_to_handle( i->array, i->count * i->nodes_per_elem );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 rval = readUtil->update_adjacencies( i->handle, i->count, i->nodes_per_elem, i->array );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 idConnectivityList.clear();
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_node_adj_elems( const mhdf_ElemDesc& group, Range* handles_out )
{
 mhdf_Status status;
 ErrorCode rval;
 
 CHECK_OPEN_HANDLES;
 
 hid_t table = mhdf_openConnectivitySimple( filePtr, group.handle, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 rval = read_node_adj_elems( group, table, handles_out );
 
 mhdf_closeData( filePtr, table, &status );
 if( MB_SUCCESS == rval && is_error( status ) ) MB_SET_ERR_RET_VAL( "ReadHDF5 Failure", MB_FAILURE );
 
 return rval;
}
 
ErrorCode ReadHDF5::read_node_adj_elems( const mhdf_ElemDesc& group, hid_t table_handle, Range* handles_out )
{
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 
 mhdf_Status status;
 ErrorCode rval;
 IODebugTrack debug_track( debugTrack, std::string( group.handle ) );
 
 // Copy data to local variables (makes other code clearer)
 const int node_per_elem = group.desc.vals_per_ent;
 long start_id = group.desc.start_id;
 long remaining = group.desc.count;
 const EntityType type = CN::EntityTypeFromName( group.type );
 
 // Figure out how many elements we can read in each pass
 long* const buffer = reinterpret_cast< long* >( dataBuffer );
 const long buffer_size = bufferSize / ( node_per_elem * sizeof( buffer[0] ) );
 // Read all element connectivity in buffer_size blocks
 long offset = 0;
 dbgOut.printf( 3, "Reading node-adjacent elements from \"%s\" in %ld chunks\n", group.handle,
 ( remaining + buffer_size - 1 ) / buffer_size );
 int nn = 0;
 Range::iterator hint;
 if( handles_out ) hint = handles_out->begin();
 while( remaining )
 {
 dbgOut.printf( 3, "Reading chunk %d of connectivity data for \"%s\"\n", ++nn, group.handle );
 
 // Read a block of connectivity data
 const long count = std::min( remaining, buffer_size );
 debug_track.record_io( offset, count );
 assert_range( buffer, count * node_per_elem );
 mhdf_readConnectivityWithOpt( table_handle, offset, count, H5T_NATIVE_LONG, buffer, collIO, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 offset += count;
 remaining -= count;
 
 // Count the number of elements in the block that we want,
 // zero connectivity for other elements
 long num_elem = 0;
 long* iter = buffer;
 for( long i = 0; i < count; ++i )
 {
 for( int j = 0; j < node_per_elem; ++j )
 {
 iter[j] = (long)idMap.find( iter[j] );
 if( !iter[j] )
 {
 iter[0] = 0;
 break;
 }
 }
 if( iter[0] ) ++num_elem;
 iter += node_per_elem;
 }
 
 if( !num_elem )
 {
 start_id += count;
 continue;
 }
 
 // Create elements
 EntityHandle handle;
 EntityHandle* array;
 rval = readUtil->get_element_connect( (int)num_elem, node_per_elem, type, 0, handle, array );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 // Copy all non-zero connectivity values
 iter = buffer;
 EntityHandle* iter2 = array;
 EntityHandle h = handle;
 for( long i = 0; i < count; ++i )
 {
 if( !*iter )
 {
 iter += node_per_elem;
 continue;
 }
 if( !idMap.insert( start_id + i, h++, 1 ).second ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 long* const end = iter + node_per_elem;
 for( ; iter != end; ++iter, ++iter2 )
 *iter2 = (EntityHandle)*iter;
 }
 assert( iter2 - array == num_elem * node_per_elem );
 start_id += count;
 
 rval = readUtil->update_adjacencies( handle, num_elem, node_per_elem, array );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 if( handles_out ) hint = handles_out->insert( hint, handle, handle + num_elem - 1 );
 }
 
 debug_track.all_reduce();
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_elems( int i, const Range& elems_in, Range& nodes )
{
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 dbgOut.tprintf( 1, "READING %s CONNECTIVITY (%lu elems in %lu selects)\n", fileInfo->elems[i].handle,
 (unsigned long)elems_in.size(), (unsigned long)elems_in.psize() );
 
 EntityHandle* const buffer = reinterpret_cast< EntityHandle* >( dataBuffer );
 const int node_per_elem = fileInfo->elems[i].desc.vals_per_ent;
 const size_t buffer_size = bufferSize / ( node_per_elem * sizeof( EntityHandle ) );
 
 if( elems_in.empty() ) return MB_SUCCESS;
 
 assert( (long)elems_in.front() >= fileInfo->elems[i].desc.start_id );
 assert( (long)elems_in.back() - fileInfo->elems[i].desc.start_id < fileInfo->elems[i].desc.count );
 
 // We don't support version 3 style poly element data
 if( fileInfo->elems[i].desc.vals_per_ent <= 0 ) MB_CHK_ERR( MB_TYPE_OUT_OF_RANGE );
 
 mhdf_Status status;
 hid_t table = mhdf_openConnectivitySimple( filePtr, fileInfo->elems[i].handle, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 try
 {
 ReadHDF5Dataset reader( fileInfo->elems[i].handle, table, nativeParallel, mpiComm );
 reader.set_file_ids( elems_in, fileInfo->elems[i].desc.start_id, buffer_size, handleType );
 dbgOut.printf( 3, "Reading node list in %lu chunks for \"%s\"\n", reader.get_read_count(),
 fileInfo->elems[i].handle );
 int nn = 0;
 while( !reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of \"%s\" connectivity\n", ++nn, fileInfo->elems[i].handle );
 size_t num_read;
 reader.read( buffer, num_read );
 std::sort( buffer, buffer + num_read * node_per_elem );
 num_read = std::unique( buffer, buffer + num_read * node_per_elem ) - buffer;
 copy_sorted_file_ids( buffer, num_read, nodes );
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_poly( const mhdf_ElemDesc& elems, const Range& file_ids )
{
 class PolyReader : public ReadHDF5VarLen
 {
 private:
 const EntityType type;
 ReadHDF5* readHDF5;
 
 public:
 PolyReader( EntityType elem_type, void* buffer, size_t buffer_size, ReadHDF5* owner, DebugOutput& dbg )
 : ReadHDF5VarLen( dbg, buffer, buffer_size ), type( elem_type ), readHDF5( owner )
 {
 }
 virtual ~PolyReader() {}
 ErrorCode store_data( EntityHandle file_id, void* data, long len, bool )
 {
 size_t valid;
 EntityHandle* conn = reinterpret_cast< EntityHandle* >( data );
 readHDF5->convert_id_to_handle( conn, len, valid );
 if( valid != (size_t)len ) MB_CHK_ERR( MB_ENTITY_NOT_FOUND );
 EntityHandle handle;
 ErrorCode rval = readHDF5->moab()->create_element( type, conn, len, handle );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 rval = readHDF5->insert_in_id_map( file_id, handle );
 return rval;
 }
 };
 
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 
 EntityType type = CN::EntityTypeFromName( elems.type );
 if( type == MBMAXTYPE )
 {
 MB_SET_ERR( MB_FAILURE, "Unknown element type: \"" << elems.type << "\"" );
 }
 
 hid_t handles[2];
 mhdf_Status status;
 long num_poly, num_conn, first_id;
 mhdf_openPolyConnectivity( filePtr, elems.handle, &num_poly, &num_conn, &first_id, handles, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 std::string nm( elems.handle );
 ReadHDF5Dataset offset_reader( ( nm + " offsets" ).c_str(), handles[0], nativeParallel, mpiComm, true );
 ReadHDF5Dataset connect_reader( ( nm + " data" ).c_str(), handles[1], nativeParallel, mpiComm, true );
 
 PolyReader tool( type, dataBuffer, bufferSize, this, dbgOut );
 return tool.read( offset_reader, connect_reader, file_ids, first_id, handleType );
}
 
ErrorCode ReadHDF5::delete_non_side_elements( const Range& side_ents )
{
 ErrorCode rval;
 
 // Build list of entities that we need to find the sides of
 Range explicit_ents;
 Range::iterator hint = explicit_ents.begin();
 for( IDMap::iterator i = idMap.begin(); i != idMap.end(); ++i )
 {
 EntityHandle start = i->value;
 EntityHandle end = i->value + i->count - 1;
 EntityType type = TYPE_FROM_HANDLE( start );
 assert( type == TYPE_FROM_HANDLE( end ) ); // Otherwise handle space entirely full!!
 if( type != MBVERTEX && type != MBENTITYSET ) hint = explicit_ents.insert( hint, start, end );
 }
 explicit_ents = subtract( explicit_ents, side_ents );
 
 // Figure out which entities we want to delete
 Range dead_ents( side_ents );
 Range::iterator ds, de, es;
 ds = dead_ents.lower_bound( CN::TypeDimensionMap[1].first );
 de = dead_ents.lower_bound( CN::TypeDimensionMap[2].first, ds );
 if( ds != de )
 {
 // Get subset of explicit ents of dimension greater than 1
 es = explicit_ents.lower_bound( CN::TypeDimensionMap[2].first );
 Range subset, adj;
 subset.insert( es, explicit_ents.end() );
 rval = iFace->get_adjacencies( subset, 1, false, adj, Interface::UNION );
 if( MB_SUCCESS != rval ) return rval;
 dead_ents = subtract( dead_ents, adj );
 }
 ds = dead_ents.lower_bound( CN::TypeDimensionMap[2].first );
 de = dead_ents.lower_bound( CN::TypeDimensionMap[3].first, ds );
 assert( de == dead_ents.end() );
 if( ds != de )
 {
 // Get subset of explicit ents of dimension 3
 es = explicit_ents.lower_bound( CN::TypeDimensionMap[3].first );
 Range subset, adj;
 subset.insert( es, explicit_ents.end() );
 rval = iFace->get_adjacencies( subset, 2, false, adj, Interface::UNION );
 if( MB_SUCCESS != rval ) return rval;
 dead_ents = subtract( dead_ents, adj );
 }
 
 // Now delete anything remaining in dead_ents
 dbgOut.printf( 2, "Deleting %lu elements\n", (unsigned long)dead_ents.size() );
 dbgOut.print( 4, "\tDead entities: ", dead_ents );
 rval = iFace->delete_entities( dead_ents );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 // Remove dead entities from ID map
 while( !dead_ents.empty() )
 {
 EntityHandle start = dead_ents.front();
 EntityID count = dead_ents.const_pair_begin()->second - start + 1;
 IDMap::iterator rit;
 for( rit = idMap.begin(); rit != idMap.end(); ++rit )
 if( rit->value <= start && (EntityID)( start - rit->value ) < rit->count ) break;
 if( rit == idMap.end() ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 EntityID offset = start - rit->value;
 EntityID avail = rit->count - offset;
 if( avail < count ) count = avail;
 
 dead_ents.erase( dead_ents.begin(), dead_ents.begin() + count );
 idMap.erase( rit->begin + offset, count );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_sets( const Range& file_ids )
{
 CHECK_OPEN_HANDLES;
 
 debug_barrier();
 
 mhdf_Status status;
 ErrorCode rval;
 
 const size_t num_sets = fileInfo->sets.count;
 if( !num_sets ) // If no sets at all!
 return MB_SUCCESS;
 
 // Create sets
 std::vector< unsigned > flags( file_ids.size() );
 Range::iterator si = file_ids.begin();
 for( size_t i = 0; i < flags.size(); ++i, ++si )
 flags[i] = setMeta[*si - fileInfo->sets.start_id][3] & ~(long)mhdf_SET_RANGE_BIT;
 EntityHandle start_handle;
 // the files ids could be empty, for empty partitions
 if( !file_ids.empty() )
 {
 rval = readUtil->create_entity_sets( flags.size(), &flags[0], 0, start_handle );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 rval = insert_in_id_map( file_ids, start_handle );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 // Read contents
 if( fileInfo->have_set_contents )
 {
 long len = 0;
 hid_t handle = mhdf_openSetData( filePtr, &len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 ReadHDF5Dataset dat( "set contents", handle, nativeParallel, mpiComm, true );
 rval = read_set_data( file_ids, start_handle, dat, CONTENT );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 // Read set child lists
 if( fileInfo->have_set_children )
 {
 long len = 0;
 hid_t handle = mhdf_openSetChildren( filePtr, &len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 ReadHDF5Dataset dat( "set children", handle, nativeParallel, mpiComm, true );
 rval = read_set_data( file_ids, start_handle, dat, CHILD );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 // Read set parent lists
 if( fileInfo->have_set_parents )
 {
 long len = 0;
 hid_t handle = mhdf_openSetParents( filePtr, &len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 ReadHDF5Dataset dat( "set parents", handle, nativeParallel, mpiComm, true );
 rval = read_set_data( file_ids, start_handle, dat, PARENT );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_all_set_meta()
{
 CHECK_OPEN_HANDLES;
 
 assert( !setMeta );
 const long num_sets = fileInfo->sets.count;
 if( !num_sets ) return MB_SUCCESS;
 
 mhdf_Status status;
 hid_t handle = mhdf_openSetMetaSimple( filePtr, &status );
 if( is_error( status ) )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 // Allocate extra space if we need it for data conversion
 hid_t meta_type = H5Dget_type( handle );
 size_t size = H5Tget_size( meta_type );
 if( size > sizeof( long ) )
 setMeta = new long[( num_sets * size + ( sizeof( long ) - 1 ) ) / sizeof( long )][4];
 else
 setMeta = new long[num_sets][4];
 
 // Set some parameters based on whether or not each proc reads the
 // table or only the root reads it and bcasts it to the others
 int rank = 0;
 bool bcast = false;
 hid_t ioprop = H5P_DEFAULT;
#ifdef MOAB_HAVE_MPI
 MPI_Comm comm = 0;
 if( nativeParallel )
 {
 rank = myPcomm->proc_config().proc_rank();
 comm = myPcomm->proc_config().proc_comm();
 bcast = bcastDuplicateReads;
 if( !bcast ) ioprop = collIO;
 }
#endif
 
 if( !bcast || 0 == rank )
 {
 mhdf_readSetMetaWithOpt( handle, 0, num_sets, meta_type, setMeta, ioprop, &status );
 if( is_error( status ) )
 {
 H5Tclose( meta_type );
 mhdf_closeData( filePtr, handle, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 H5Tconvert( meta_type, H5T_NATIVE_LONG, num_sets * 4, setMeta, 0, H5P_DEFAULT );
 }
 mhdf_closeData( filePtr, handle, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 H5Tclose( meta_type );
 
 if( bcast )
 {
#ifdef MOAB_HAVE_MPI
 int ierr = MPI_Bcast( (void*)setMeta, num_sets * 4, MPI_LONG, 0, comm );
 if( MPI_SUCCESS != ierr ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
#else
 assert( rank == 0 ); // If not MPI, then only one proc
#endif
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_set_ids_recursive( Range& sets_in_out, bool contained_sets, bool child_sets )
{
 CHECK_OPEN_HANDLES;
 mhdf_Status status;
 
 if( !fileInfo->have_set_children ) child_sets = false;
 if( !fileInfo->have_set_contents ) contained_sets = false;
 if( !child_sets && !contained_sets ) return MB_SUCCESS;
 
 // Open data tables
 if( fileInfo->sets.count == 0 )
 {
 assert( sets_in_out.empty() );
 return MB_SUCCESS;
 }
 
 if( !contained_sets && !child_sets ) return MB_SUCCESS;
 
 ReadHDF5Dataset cont( "set contents", false, mpiComm );
 ReadHDF5Dataset child( "set children", false, mpiComm );
 
 if( contained_sets )
 {
 long content_len = 0;
 hid_t content_handle = mhdf_openSetData( filePtr, &content_len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 try
 {
 cont.init( content_handle, true );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 
 if( child_sets )
 {
 long child_len = 0;
 hid_t child_handle = mhdf_openSetChildren( filePtr, &child_len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 try
 {
 child.init( child_handle, true );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 
 ErrorCode rval = MB_SUCCESS;
 Range children, new_children( sets_in_out );
 int iteration_count = 0;
 do
 {
 ++iteration_count;
 dbgOut.tprintf( 2, "Iteration %d of read_set_ids_recursive\n", iteration_count );
 children.clear();
 if( child_sets )
 {
 rval = read_set_data( new_children, 0, child, CHILD, &children );
 if( MB_SUCCESS != rval ) break;
 }
 if( contained_sets )
 {
 rval = read_set_data( new_children, 0, cont, CONTENT, &children );
 // Remove any non-set values
 Range::iterator it = children.lower_bound( fileInfo->sets.start_id );
 children.erase( children.begin(), it );
 it = children.lower_bound( fileInfo->sets.start_id + fileInfo->sets.count );
 children.erase( it, children.end() );
 if( MB_SUCCESS != rval ) break;
 }
 new_children = subtract( children, sets_in_out );
 dbgOut.print_ints( 2, "Adding additional contained/child sets", new_children );
 sets_in_out.merge( new_children );
 } while( !new_children.empty() );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::find_sets_containing( Range& sets_out, bool read_set_containing_parents )
{
 ErrorCode rval;
 mhdf_Status status;
 
 CHECK_OPEN_HANDLES;
 
 if( !fileInfo->have_set_contents ) return MB_SUCCESS;
 assert( fileInfo->sets.count );
 
 // Open data tables
 long content_len = 0;
 hid_t content_handle = mhdf_openSetData( filePtr, &content_len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 hid_t data_type = H5Dget_type( content_handle );
 
 rval = find_sets_containing( content_handle, data_type, content_len, read_set_containing_parents, sets_out );
 
 H5Tclose( data_type );
 
 mhdf_closeData( filePtr, content_handle, &status );
 if( MB_SUCCESS == rval && is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 return rval;
}
 
static bool set_map_intersect( bool ranged,
 const long* contents,
 int content_len,
 const RangeMap< long, EntityHandle >& id_map )
{
 if( ranged )
 {
 if( !content_len || id_map.empty() ) return false;
 
 const long* j = contents;
 const long* const end = contents + content_len;
 assert( content_len % 2 == 0 );
 while( j != end )
 {
 long start = *( j++ );
 long count = *( j++ );
 if( id_map.intersects( start, count ) ) return true;
 }
 }
 else
 {
 const long* const end = contents + content_len;
 for( const long* i = contents; i != end; ++i )
 if( id_map.exists( *i ) ) return true;
 }
 
 return false;
}
 
struct SetContOffComp
{
 bool operator()( const long a1[4], const long a2[4] )
 {
 return a1[ReadHDF5::CONTENT] < a2[0];
 }
};
 
ErrorCode ReadHDF5::find_sets_containing( hid_t contents_handle,
 hid_t content_type,
 long contents_len,
 bool read_set_containing_parents,
 Range& file_ids )
{
 CHECK_OPEN_HANDLES;
 
 // Scan all set contents data
 
 const size_t content_size = H5Tget_size( content_type );
 const long num_sets = fileInfo->sets.count;
 dbgOut.printf( 2, "Searching contents of %ld\n", num_sets );
 mhdf_Status status;
 
 int rank = 0;
 bool bcast = false;
#ifdef MOAB_HAVE_MPI
 MPI_Comm comm = 0;
 if( nativeParallel )
 {
 rank = myPcomm->proc_config().proc_rank();
 comm = myPcomm->proc_config().proc_comm();
 bcast = bcastDuplicateReads;
 }
#endif
 
 // Check offsets so that we don't read past end of table or
 // walk off end of array.
 long prev = -1;
 for( long i = 0; i < num_sets; ++i )
 {
 if( setMeta[i][CONTENT] < prev )
 {
 std::cerr << "Invalid data in set contents offsets at position " << i << ": index " << setMeta[i][CONTENT]
 << " is less than previous index " << prev << std::endl;
 std::cerr.flush();
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 prev = setMeta[i][CONTENT];
 }
 if( setMeta[num_sets - 1][CONTENT] >= contents_len )
 {
 std::cerr << "Maximum set content index " << setMeta[num_sets - 1][CONTENT]
 << " exceeds contents table length of " << contents_len << std::endl;
 std::cerr.flush();
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 // Set up buffer for reading set contents
 long* const content_buffer = (long*)dataBuffer;
 const long content_len = bufferSize / std::max( content_size, sizeof( long ) );
 
 // Scan set table
 Range::iterator hint = file_ids.begin();
 Range tmp_range;
 long prev_idx = -1;
 int mm = 0;
 long sets_offset = 0;
 long temp_content[4];
 while( sets_offset < num_sets )
 {
 temp_content[0] = content_len + prev_idx;
 long sets_count =
 std::lower_bound( setMeta + sets_offset, setMeta + num_sets, temp_content, SetContOffComp() ) - setMeta -
 sets_offset;
 assert( sets_count >= 0 && sets_offset + sets_count <= num_sets );
 if( !sets_count )
 { // Contents of single set don't fit in buffer
 long content_remaining = setMeta[sets_offset][CONTENT] - prev_idx;
 long content_offset = prev_idx + 1;
 while( content_remaining )
 {
 long content_count = content_len < content_remaining ? 2 * ( content_len / 2 ) : content_remaining;
 assert_range( content_buffer, content_count );
 dbgOut.printf( 3, "Reading chunk %d (%ld values) from set contents table\n", ++mm, content_count );
 if( !bcast || 0 == rank )
 {
 if( !bcast )
 mhdf_readSetDataWithOpt( contents_handle, content_offset, content_count, content_type,
 content_buffer, collIO, &status );
 else
 mhdf_readSetData( contents_handle, content_offset, content_count, content_type, content_buffer,
 &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 H5Tconvert( content_type, H5T_NATIVE_LONG, content_count, content_buffer, 0, H5P_DEFAULT );
 }
 if( bcast )
 {
#ifdef MOAB_HAVE_MPI
 int ierr = MPI_Bcast( content_buffer, content_count, MPI_LONG, 0, comm );
 if( MPI_SUCCESS != ierr ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
#else
 assert( rank == 0 ); // If not MPI, then only one proc
#endif
 }
 
 if( read_set_containing_parents )
 {
 tmp_range.clear();
 if( setMeta[sets_offset][3] & mhdf_SET_RANGE_BIT )
 tmp_range.insert( *content_buffer, *( content_buffer + 1 ) );
 else
 std::copy( content_buffer, content_buffer + content_count, range_inserter( tmp_range ) );
 tmp_range = intersect( tmp_range, file_ids );
 }
 
 if( !tmp_range.empty() || set_map_intersect( setMeta[sets_offset][3] & mhdf_SET_RANGE_BIT,
 content_buffer, content_count, idMap ) )
 {
 long id = fileInfo->sets.start_id + sets_offset;
 hint = file_ids.insert( hint, id, id );
 if( !nativeParallel ) // Don't stop if doing READ_PART because we need to read
 // collectively
 break;
 }
 content_remaining -= content_count;
 content_offset += content_count;
 }
 prev_idx = setMeta[sets_offset][CONTENT];
 sets_count = 1;
 }
 else if( long read_num = setMeta[sets_offset + sets_count - 1][CONTENT] - prev_idx )
 {
 assert( sets_count > 0 );
 assert_range( content_buffer, read_num );
 dbgOut.printf( 3, "Reading chunk %d (%ld values) from set contents table\n", ++mm, read_num );
 if( !bcast || 0 == rank )
 {
 if( !bcast )
 mhdf_readSetDataWithOpt( contents_handle, prev_idx + 1, read_num, content_type, content_buffer,
 collIO, &status );
 else
 mhdf_readSetData( contents_handle, prev_idx + 1, read_num, content_type, content_buffer, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 H5Tconvert( content_type, H5T_NATIVE_LONG, read_num, content_buffer, 0, H5P_DEFAULT );
 }
 if( bcast )
 {
#ifdef MOAB_HAVE_MPI
 int ierr = MPI_Bcast( content_buffer, read_num, MPI_LONG, 0, comm );
 if( MPI_SUCCESS != ierr ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
#else
 assert( rank == 0 ); // If not MPI, then only one proc
#endif
 }
 
 long* buff_iter = content_buffer;
 for( long i = 0; i < sets_count; ++i )
 {
 long set_size = setMeta[i + sets_offset][CONTENT] - prev_idx;
 prev_idx += set_size;
 
 // Check whether contents include set already being loaded
 if( read_set_containing_parents )
 {
 tmp_range.clear();
 if( setMeta[sets_offset + i][3] & mhdf_SET_RANGE_BIT )
 {
 // put in tmp_range the contents on the set
 // file_ids contain at this points only other sets
 const long* j = buff_iter;
 const long* const end = buff_iter + set_size;
 assert( set_size % 2 == 0 );
 while( j != end )
 {
 long start = *( j++ );
 long count = *( j++ );
 tmp_range.insert( start, start + count - 1 );
 }
 }
 else
 std::copy( buff_iter, buff_iter + set_size, range_inserter( tmp_range ) );
 tmp_range = intersect( tmp_range, file_ids );
 }
 
 if( !tmp_range.empty() ||
 set_map_intersect( setMeta[sets_offset + i][3] & mhdf_SET_RANGE_BIT, buff_iter, set_size, idMap ) )
 {
 long id = fileInfo->sets.start_id + sets_offset + i;
 hint = file_ids.insert( hint, id, id );
 }
 buff_iter += set_size;
 }
 }
 
 sets_offset += sets_count;
 }
 
 return MB_SUCCESS;
}
 
static Range::iterator copy_set_contents( Range::iterator hint,
 int ranged,
 EntityHandle* contents,
 long length,
 Range& results )
{
 if( ranged )
 {
 assert( length % 2 == 0 );
 for( long i = 0; i < length; i += 2 )
 hint = results.insert( hint, contents[i], contents[i] + contents[i + 1] - 1 );
 }
 else
 {
 std::sort( contents, contents + length );
 for( long i = 0; i < length; ++i )
 hint = results.insert( hint, contents[i] );
 }
 return hint;
}
 
ErrorCode ReadHDF5::read_set_data( const Range& set_file_ids,
 EntityHandle start_handle,
 ReadHDF5Dataset& data,
 SetMode mode,
 Range* file_ids_out )
{
 ErrorCode rval;
 Range::const_pair_iterator pi;
 Range::iterator out_hint;
 if( file_ids_out ) out_hint = file_ids_out->begin();
 
 // Construct range of offsets into data table at which to read
 // Note: all offsets are incremented by TWEAK because Range cannot
 // store zeros.
 const long TWEAK = 1;
 Range data_offsets;
 Range::iterator hint = data_offsets.begin();
 pi = set_file_ids.const_pair_begin();
 if( (long)pi->first == fileInfo->sets.start_id )
 {
 long second = pi->second - fileInfo->sets.start_id;
 if( setMeta[second][mode] >= 0 ) hint = data_offsets.insert( hint, TWEAK, setMeta[second][mode] + TWEAK );
 ++pi;
 }
 for( ; pi != set_file_ids.const_pair_end(); ++pi )
 {
 long first = pi->first - fileInfo->sets.start_id;
 long second = pi->second - fileInfo->sets.start_id;
 long idx1 = setMeta[first - 1][mode] + 1;
 long idx2 = setMeta[second][mode];
 if( idx2 >= idx1 ) hint = data_offsets.insert( hint, idx1 + TWEAK, idx2 + TWEAK );
 }
 try
 {
 data.set_file_ids( data_offsets, TWEAK, bufferSize / sizeof( EntityHandle ), handleType );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 return MB_FAILURE;
 }
 
 // We need to increment this for each processed set because
 // the sets were created in the order of the ids in file_ids.
 EntityHandle h = start_handle;
 
 const long ranged_flag = ( mode == CONTENT ) ? mhdf_SET_RANGE_BIT : 0;
 
 std::vector< EntityHandle > partial; // For when we read only part of the contents of a set/entity
 Range::const_iterator fileid_iter = set_file_ids.begin();
 EntityHandle* buffer = reinterpret_cast< EntityHandle* >( dataBuffer );
 size_t count, offset;
 
 int nn = 0;
 /*
 #ifdef MOAB_HAVE_MPI
 if (nativeParallel && mode==CONTENT && myPcomm->proc_config().proc_size()>1 &&
 data_offsets.empty())
 {
 MB_SET_ERR_CONT( "ReadHDF5 Failure: Attempt reading an empty dataset on proc " <<
 myPcomm->proc_config().proc_rank());
 MPI_Abort(myPcomm->proc_config().proc_comm(), 1);
 }
 #endif
 */
 if( ( 1 >= set_file_ids.size() ) && ( data.done() ) && moab::ReadHDF5::CONTENT == mode )
 // do at least one null read, it is needed in parallel
 data.null_read();
 
 while( !data.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data.get_debug_desc() );
 try
 {
 data.read( buffer, count );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 return MB_FAILURE;
 }
 
 // Assert not appropriate here - I might have treated all my file ids, but maybe
 // another proc hasn't; for me, count will be zero, so I won't do anything, but
 // I still need to go through the motions to make the read work
 
 // Handle 'special' case where we read some, but not all
 // of the data for an entity during the last iteration.
 offset = 0;
 if( !partial.empty() )
 { // Didn't read all of previous entity
 assert( fileid_iter != set_file_ids.end() );
 size_t num_prev = partial.size();
 size_t idx = *fileid_iter - fileInfo->sets.start_id;
 size_t len = idx ? setMeta[idx][mode] - setMeta[idx - 1][mode] : setMeta[idx][mode] + 1;
 offset = len - num_prev;
 if( offset > count )
 { // Still don't have all
 partial.insert( partial.end(), buffer, buffer + count );
 continue;
 }
 
 partial.insert( partial.end(), buffer, buffer + offset );
 if( file_ids_out )
 {
 out_hint = copy_set_contents( out_hint, setMeta[idx][3] & ranged_flag, &partial[0], partial.size(),
 *file_ids_out );
 }
 else
 {
 switch( mode )
 {
 size_t valid;
 case CONTENT:
 if( setMeta[idx][3] & ranged_flag )
 {
 if( len % 2 ) MB_CHK_ERR( MB_INDEX_OUT_OF_RANGE );
 Range range;
 convert_range_to_handle( &partial[0], len / 2, range );
 rval = moab()->add_entities( h, range );
 }
 else
 {
 convert_id_to_handle( &partial[0], len, valid );
 rval = moab()->add_entities( h, &partial[0], valid );
 }
 break;
 case CHILD:
 convert_id_to_handle( &partial[0], len, valid );
 rval = moab()->add_child_meshsets( h, &partial[0], valid );
 break;
 case PARENT:
 convert_id_to_handle( &partial[0], len, valid );
 rval = moab()->add_parent_meshsets( h, &partial[0], valid );
 break;
 default:
 break;
 }
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 ++fileid_iter;
 ++h;
 partial.clear();
 }
 
 // Process contents for all entities for which we
 // have read the complete list
 while( offset < count )
 {
 assert( fileid_iter != set_file_ids.end() );
 size_t idx = *fileid_iter - fileInfo->sets.start_id;
 size_t len = idx ? setMeta[idx][mode] - setMeta[idx - 1][mode] : setMeta[idx][mode] + 1;
 // If we did not read all of the final entity,
 // store what we did read to be processed in the
 // next iteration
 if( offset + len > count )
 {
 partial.insert( partial.end(), buffer + offset, buffer + count );
 break;
 }
 
 if( file_ids_out )
 {
 out_hint =
 copy_set_contents( out_hint, setMeta[idx][3] & ranged_flag, buffer + offset, len, *file_ids_out );
 }
 else
 {
 switch( mode )
 {
 size_t valid;
 case CONTENT:
 if( setMeta[idx][3] & ranged_flag )
 {
 if( len % 2 ) MB_CHK_ERR( MB_INDEX_OUT_OF_RANGE );
 Range range;
 convert_range_to_handle( buffer + offset, len / 2, range );
 rval = moab()->add_entities( h, range );
 }
 else
 {
 convert_id_to_handle( buffer + offset, len, valid );
 rval = moab()->add_entities( h, buffer + offset, valid );
 }
 break;
 case CHILD:
 convert_id_to_handle( buffer + offset, len, valid );
 rval = moab()->add_child_meshsets( h, buffer + offset, valid );
 break;
 case PARENT:
 convert_id_to_handle( buffer + offset, len, valid );
 rval = moab()->add_parent_meshsets( h, buffer + offset, valid );
 break;
 default:
 break;
 }
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 ++fileid_iter;
 ++h;
 offset += len;
 }
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::get_set_contents( const Range& sets, Range& file_ids )
{
 CHECK_OPEN_HANDLES;
 
 if( !fileInfo->have_set_contents ) return MB_SUCCESS;
 dbgOut.tprint( 2, "Reading set contained file IDs\n" );
 try
 {
 mhdf_Status status;
 long content_len;
 hid_t contents = mhdf_openSetData( filePtr, &content_len, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 ReadHDF5Dataset data( "set contents", contents, nativeParallel, mpiComm, true );
 
 return read_set_data( sets, 0, data, CONTENT, &file_ids );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
}
 
ErrorCode ReadHDF5::read_adjacencies( hid_t table, long table_len )
{
 CHECK_OPEN_HANDLES;
 
 ErrorCode rval;
 mhdf_Status status;
 
 debug_barrier();
 
 hid_t read_type = H5Dget_type( table );
 if( read_type < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 const bool convert = !H5Tequal( read_type, handleType );
 
 EntityHandle* buffer = (EntityHandle*)dataBuffer;
 size_t chunk_size = bufferSize / H5Tget_size( read_type );
 size_t remaining = table_len;
 size_t left_over = 0;
 size_t offset = 0;
 dbgOut.printf( 3, "Reading adjacency list in %lu chunks\n",
 (unsigned long)( remaining + chunk_size - 1 ) / chunk_size );
 int nn = 0;
 while( remaining )
 {
 dbgOut.printf( 3, "Reading chunk %d of adjacency list\n", ++nn );
 
 size_t count = std::min( chunk_size, remaining );
 count -= left_over;
 remaining -= count;
 
 assert_range( buffer + left_over, count );
 mhdf_readAdjacencyWithOpt( table, offset, count, read_type, buffer + left_over, collIO, &status );
 if( is_error( status ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 if( convert )
 {
 herr_t err = H5Tconvert( read_type, handleType, count, buffer + left_over, 0, H5P_DEFAULT );
 if( err < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 EntityHandle* iter = buffer;
 EntityHandle* end = buffer + count + left_over;
 while( end - iter >= 3 )
 {
 EntityHandle h = idMap.find( *iter++ );
 EntityHandle count2 = *iter++;
 if( !h )
 {
 iter += count2;
 continue;
 }
 
 if( count2 < 1 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 if( end < count2 + iter )
 {
 iter -= 2;
 break;
 }
 
 size_t valid;
 convert_id_to_handle( iter, count2, valid, idMap );
 rval = iFace->add_adjacencies( h, iter, valid, false );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 iter += count2;
 }
 
 left_over = end - iter;
 assert_range( (char*)buffer, left_over );
 assert_range( (char*)iter, left_over );
 memmove( buffer, iter, left_over );
 }
 
 assert( !left_over ); // Unexpected truncation of data
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_tag( int tag_index )
{
 CHECK_OPEN_HANDLES;
 
 dbgOut.tprintf( 2, "Reading tag \"%s\"\n", fileInfo->tags[tag_index].name );
 
 debug_barrier();
 
 ErrorCode rval;
 mhdf_Status status;
 Tag tag = 0;
 hid_t read_type = -1;
 bool table_type;
 rval = create_tag( fileInfo->tags[tag_index], tag, read_type );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( fileInfo->tags[tag_index].have_sparse )
 {
 hid_t handles[3];
 long num_ent, num_val;
 mhdf_openSparseTagData( filePtr, fileInfo->tags[tag_index].name, &num_ent, &num_val, handles, &status );
 if( is_error( status ) )
 {
 if( read_type ) H5Tclose( read_type );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 table_type = false;
 if( read_type == 0 )
 {
 read_type = H5Dget_type( handles[1] );
 if( read_type == 0 )
 {
 mhdf_closeData( filePtr, handles[0], &status );
 mhdf_closeData( filePtr, handles[0], &status );
 if( fileInfo->tags[tag_index].size <= 0 ) mhdf_closeData( filePtr, handles[2], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 table_type = true;
 }
 
 if( fileInfo->tags[tag_index].size > 0 )
 {
 dbgOut.printf( 2, "Reading sparse data for tag \"%s\"\n", fileInfo->tags[tag_index].name );
 rval = read_sparse_tag( tag, read_type, handles[0], handles[1], num_ent );
 }
 else
 {
 dbgOut.printf( 2, "Reading var-len sparse data for tag \"%s\"\n", fileInfo->tags[tag_index].name );
 rval = read_var_len_tag( tag, read_type, handles[0], handles[1], handles[2], num_ent, num_val );
 }
 
 if( table_type )
 {
 H5Tclose( read_type );
 read_type = 0;
 }
 
 mhdf_closeData( filePtr, handles[0], &status );
 if( MB_SUCCESS == rval && is_error( status ) ) rval = MB_FAILURE;
 mhdf_closeData( filePtr, handles[1], &status );
 if( MB_SUCCESS == rval && is_error( status ) ) rval = MB_FAILURE;
 if( fileInfo->tags[tag_index].size <= 0 )
 {
 mhdf_closeData( filePtr, handles[2], &status );
 if( MB_SUCCESS == rval && is_error( status ) ) rval = MB_FAILURE;
 }
 if( MB_SUCCESS != rval )
 {
 if( read_type ) H5Tclose( read_type );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 for( int j = 0; j < fileInfo->tags[tag_index].num_dense_indices; ++j )
 {
 long count;
 const char* name = 0;
 mhdf_EntDesc* desc;
 int elem_idx = fileInfo->tags[tag_index].dense_elem_indices[j];
 if( elem_idx == -2 )
 {
 desc = &fileInfo->sets;
 name = mhdf_set_type_handle();
 }
 else if( elem_idx == -1 )
 {
 desc = &fileInfo->nodes;
 name = mhdf_node_type_handle();
 }
 else if( elem_idx >= 0 && elem_idx < fileInfo->num_elem_desc )
 {
 desc = &fileInfo->elems[elem_idx].desc;
 name = fileInfo->elems[elem_idx].handle;
 }
 else
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 dbgOut.printf( 2, "Read dense data block for tag \"%s\" on \"%s\"\n", fileInfo->tags[tag_index].name, name );
 
 hid_t handle = mhdf_openDenseTagData( filePtr, fileInfo->tags[tag_index].name, name, &count, &status );
 if( is_error( status ) )
 {
 rval = MB_FAILURE; // rval = error(MB_FAILURE);
 break;
 }
 
 if( count > desc->count )
 {
 mhdf_closeData( filePtr, handle, &status );
 MB_SET_ERR( MB_FAILURE,
 "Invalid data length for dense tag data: " << name << "/" << fileInfo->tags[tag_index].name );
 }
 
 table_type = false;
 if( read_type == 0 )
 {
 read_type = H5Dget_type( handle );
 if( read_type == 0 )
 {
 mhdf_closeData( filePtr, handle, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 table_type = true;
 }
 
 rval = read_dense_tag( tag, name, read_type, handle, desc->start_id, count );
 
 if( table_type )
 {
 H5Tclose( read_type );
 read_type = 0;
 }
 
 mhdf_closeData( filePtr, handle, &status );
 if( MB_SUCCESS != rval ) break;
 if( is_error( status ) )
 {
 rval = MB_FAILURE;
 break;
 }
 }
 
 if( read_type ) H5Tclose( read_type );
 return rval;
}
 
ErrorCode ReadHDF5::create_tag( const mhdf_TagDesc& info, Tag& handle, hid_t& hdf_type )
{
 CHECK_OPEN_HANDLES;
 
 ErrorCode rval;
 mhdf_Status status;
 TagType storage;
 DataType mb_type;
 bool re_read_default = false;
 
 switch( info.storage )
 {
 case mhdf_DENSE_TYPE:
 storage = MB_TAG_DENSE;
 break;
 case mhdf_SPARSE_TYPE:
 storage = MB_TAG_SPARSE;
 break;
 case mhdf_BIT_TYPE:
 storage = MB_TAG_BIT;
 break;
 case mhdf_MESH_TYPE:
 storage = MB_TAG_MESH;
 break;
 default:
 MB_SET_ERR( MB_FAILURE, "Invalid storage type for tag '" << info.name << "': " << info.storage );
 }
 
 // Type-specific stuff
 if( info.type == mhdf_BITFIELD )
 {
 if( info.size < 1 || info.size > 8 )
 {
 MB_SET_ERR( MB_FAILURE, "Invalid bit tag: class is MB_TAG_BIT, num bits = " << info.size );
 }
 hdf_type = H5Tcopy( H5T_NATIVE_B8 );
 mb_type = MB_TYPE_BIT;
 if( hdf_type < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 else if( info.type == mhdf_OPAQUE )
 {
 mb_type = MB_TYPE_OPAQUE;
 
 // Check for user-provided type
 Tag type_handle;
 std::string tag_type_name = "__hdf5_tag_type_";
 tag_type_name += info.name;
 rval = iFace->tag_get_handle( tag_type_name.c_str(), sizeof( hid_t ), MB_TYPE_OPAQUE, type_handle );
 if( MB_SUCCESS == rval )
 {
 EntityHandle root = 0;
 rval = iFace->tag_get_data( type_handle, &root, 1, &hdf_type );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 hdf_type = H5Tcopy( hdf_type );
 re_read_default = true;
 }
 else if( MB_TAG_NOT_FOUND == rval )
 {
 hdf_type = 0;
 }
 else
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 if( hdf_type < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 else
 {
 switch( info.type )
 {
 case mhdf_INTEGER:
 hdf_type = H5T_NATIVE_INT;
 mb_type = MB_TYPE_INTEGER;
 break;
 case mhdf_FLOAT:
 hdf_type = H5T_NATIVE_DOUBLE;
 mb_type = MB_TYPE_DOUBLE;
 break;
 case mhdf_BOOLEAN:
 hdf_type = H5T_NATIVE_UINT;
 mb_type = MB_TYPE_INTEGER;
 break;
 case mhdf_ENTITY_ID:
 hdf_type = handleType;
 mb_type = MB_TYPE_HANDLE;
 break;
 default:
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 if( info.size > 1 )
 { // Array
 hsize_t tmpsize = info.size;
#if defined( H5Tarray_create_vers ) && H5Tarray_create_vers > 1
 hdf_type = H5Tarray_create2( hdf_type, 1, &tmpsize );
#else
 hdf_type = H5Tarray_create( hdf_type, 1, &tmpsize, NULL );
#endif
 }
 else
 {
 hdf_type = H5Tcopy( hdf_type );
 }
 if( hdf_type < 0 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 // If default or global/mesh value in file, read it.
 if( info.default_value || info.global_value )
 {
 if( re_read_default )
 {
 mhdf_getTagValues( filePtr, info.name, hdf_type, info.default_value, info.global_value, &status );
 if( mhdf_isError( &status ) )
 {
 if( hdf_type ) H5Tclose( hdf_type );
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 }
 
 if( MB_TYPE_HANDLE == mb_type )
 {
 if( info.default_value )
 {
 rval = convert_id_to_handle( (EntityHandle*)info.default_value, info.default_value_size );
 if( MB_SUCCESS != rval )
 {
 if( hdf_type ) H5Tclose( hdf_type );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 if( info.global_value )
 {
 rval = convert_id_to_handle( (EntityHandle*)info.global_value, info.global_value_size );
 if( MB_SUCCESS != rval )
 {
 if( hdf_type ) H5Tclose( hdf_type );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 }
 }
 
 // Get tag handle, creating if necessary
 if( info.size < 0 )
 rval = iFace->tag_get_handle( info.name, info.default_value_size, mb_type, handle,
 storage | MB_TAG_CREAT | MB_TAG_VARLEN | MB_TAG_DFTOK, info.default_value );
 else
 rval = iFace->tag_get_handle( info.name, info.size, mb_type, handle, storage | MB_TAG_CREAT | MB_TAG_DFTOK,
 info.default_value );
 if( MB_SUCCESS != rval )
 {
 if( hdf_type ) H5Tclose( hdf_type );
 MB_SET_ERR( MB_FAILURE, "Tag type in file does not match type in database for \"" << info.name << "\"" );
 }
 
 if( info.global_value )
 {
 EntityHandle root = 0;
 if( info.size > 0 )
 { // Fixed-length tag
 rval = iFace->tag_set_data( handle, &root, 1, info.global_value );
 }
 else
 {
 int tag_size = info.global_value_size;
 rval = iFace->tag_set_by_ptr( handle, &root, 1, &info.global_value, &tag_size );
 }
 if( MB_SUCCESS != rval )
 {
 if( hdf_type ) H5Tclose( hdf_type );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_dense_tag( Tag tag_handle,
 const char* ent_name,
 hid_t hdf_read_type,
 hid_t data,
 long start_id,
 long num_values )
{
 CHECK_OPEN_HANDLES;
 
 ErrorCode rval;
 DataType mb_type;
 
 rval = iFace->tag_get_data_type( tag_handle, mb_type );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 int read_size;
 rval = iFace->tag_get_bytes( tag_handle, read_size );
 if( MB_SUCCESS != rval ) // Wrong function for variable-length tags
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 // if (MB_TYPE_BIT == mb_type)
 // read_size = (read_size + 7) / 8; // Convert bits to bytes, plus 7 for ceiling
 
 if( hdf_read_type )
 { // If not opaque
 hsize_t hdf_size = H5Tget_size( hdf_read_type );
 if( hdf_size != (hsize_t)read_size ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 // Get actual entities read from file
 Range file_ids, handles;
 Range::iterator f_ins = file_ids.begin(), h_ins = handles.begin();
 IDMap::iterator l, u;
 l = idMap.lower_bound( start_id );
 u = idMap.lower_bound( start_id + num_values - 1 );
 if( l != idMap.end() && start_id + num_values > l->begin )
 {
 if( l == u )
 {
 size_t beg = std::max( start_id, l->begin );
 size_t end = std::min( start_id + num_values, u->begin + u->count ) - 1;
 f_ins = file_ids.insert( f_ins, beg, end );
 h_ins = handles.insert( h_ins, l->value + ( beg - l->begin ), l->value + ( end - l->begin ) );
 }
 else
 {
 size_t beg = std::max( start_id, l->begin );
 f_ins = file_ids.insert( f_ins, beg, l->begin + l->count - 1 );
 h_ins = handles.insert( h_ins, l->value + ( beg - l->begin ), l->value + l->count - 1 );
 for( ++l; l != u; ++l )
 {
 f_ins = file_ids.insert( f_ins, l->begin, l->begin + l->count - 1 );
 h_ins = handles.insert( h_ins, l->value, l->value + l->count - 1 );
 }
 if( u != idMap.end() && u->begin < start_id + num_values )
 {
 size_t end = std::min( start_id + num_values, u->begin + u->count - 1 );
 f_ins = file_ids.insert( f_ins, u->begin, end );
 h_ins = handles.insert( h_ins, u->value, u->value + end - u->begin );
 }
 }
 }
 
 // Given that all of the entities for this dense tag data should
 // have been created as a single contiguous block, the resulting
 // MOAB handle range should be contiguous.
 // THE ABOVE IS NOT NECESSARILY TRUE. SOMETIMES LOWER-DIMENSION
 // ENTS ARE READ AND THEN DELETED FOR PARTIAL READS.
 // assert(handles.empty() || handles.size() == (handles.back() - handles.front() + 1));
 
 std::string tn( "<error>" );
 iFace->tag_get_name( tag_handle, tn );
 tn += " data for ";
 tn += ent_name;
 try
 {
 h_ins = handles.begin();
 ReadHDF5Dataset reader( tn.c_str(), data, nativeParallel, mpiComm, false );
 long buffer_size = bufferSize / read_size;
 reader.set_file_ids( file_ids, start_id, buffer_size, hdf_read_type );
 dbgOut.printf( 3, "Reading dense data for tag \"%s\" and group \"%s\" in %lu chunks\n", tn.c_str(), ent_name,
 reader.get_read_count() );
 int nn = 0;
 while( !reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of \"%s\" data\n", ++nn, tn.c_str() );
 
 size_t count;
 reader.read( dataBuffer, count );
 
 if( MB_TYPE_HANDLE == mb_type )
 {
 rval = convert_id_to_handle( (EntityHandle*)dataBuffer, count * read_size / sizeof( EntityHandle ) );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 Range ents;
 Range::iterator end = h_ins;
 end += count;
 ents.insert( h_ins, end );
 h_ins = end;
 
 rval = iFace->tag_set_data( tag_handle, ents, dataBuffer );
 if( MB_SUCCESS != rval )
 {
 dbgOut.printf( 1, "Internal error setting data for tag \"%s\"\n", tn.c_str() );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 dbgOut.printf( 1, "Internal error reading dense data for tag \"%s\"\n", tn.c_str() );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
// Read entire ID table and for those file IDs corresponding
// to entities that we have read from the file add both the
// offset into the offset range and the handle into the handle
// range. If handles are not ordered, switch to using a vector.
ErrorCode ReadHDF5::read_sparse_tag_indices( const char* name,
 hid_t id_table,
 EntityHandle start_offset, // Can't put zero in a Range
 Range& offset_range,
 Range& handle_range,
 std::vector< EntityHandle >& handle_vect )
{
 CHECK_OPEN_HANDLES;
 
 offset_range.clear();
 handle_range.clear();
 handle_vect.clear();
 
 ErrorCode rval;
 Range::iterator handle_hint = handle_range.begin();
 Range::iterator offset_hint = offset_range.begin();
 
 EntityHandle* idbuf = (EntityHandle*)dataBuffer;
 size_t idbuf_size = bufferSize / sizeof( EntityHandle );
 
 std::string tn( name );
 tn += " indices";
 
 assert( start_offset > 0 ); // Can't put zero in a Range
 try
 {
 ReadHDF5Dataset id_reader( tn.c_str(), id_table, nativeParallel, mpiComm, false );
 id_reader.set_all_file_ids( idbuf_size, handleType );
 size_t offset = start_offset;
 dbgOut.printf( 3, "Reading file ids for sparse tag \"%s\" in %lu chunks\n", name, id_reader.get_read_count() );
 int nn = 0;
 while( !id_reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of \"%s\" IDs\n", ++nn, name );
 size_t count;
 id_reader.read( idbuf, count );
 
 rval = convert_id_to_handle( idbuf, count );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 // idbuf will now contain zero-valued handles for those
 // tag values that correspond to entities we are not reading
 // from the file.
 for( size_t i = 0; i < count; ++i )
 {
 if( idbuf[i] )
 {
 offset_hint = offset_range.insert( offset_hint, offset + i );
 if( !handle_vect.empty() )
 {
 handle_vect.push_back( idbuf[i] );
 }
 else if( handle_range.empty() || idbuf[i] > handle_range.back() )
 {
 handle_hint = handle_range.insert( handle_hint, idbuf[i] );
 }
 else
 {
 handle_vect.resize( handle_range.size() );
 std::copy( handle_range.begin(), handle_range.end(), handle_vect.begin() );
 handle_range.clear();
 handle_vect.push_back( idbuf[i] );
 dbgOut.print( 2, "Switching to unordered list for tag handle list\n" );
 }
 }
 }
 
 offset += count;
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_sparse_tag( Tag tag_handle,
 hid_t hdf_read_type,
 hid_t id_table,
 hid_t value_table,
 long /*num_values*/ )
{
 CHECK_OPEN_HANDLES;
 
 // Read entire ID table and for those file IDs corresponding
 // to entities that we have read from the file add both the
 // offset into the offset range and the handle into the handle
 // range. If handles are not ordered, switch to using a vector.
 const EntityHandle base_offset = 1; // Can't put zero in a Range
 std::vector< EntityHandle > handle_vect;
 Range handle_range, offset_range;
 std::string tn( "<error>" );
 iFace->tag_get_name( tag_handle, tn );
 ErrorCode rval =
 read_sparse_tag_indices( tn.c_str(), id_table, base_offset, offset_range, handle_range, handle_vect );
 if( MB_SUCCESS != rval ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 DataType mbtype;
 rval = iFace->tag_get_data_type( tag_handle, mbtype );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 int read_size;
 rval = iFace->tag_get_bytes( tag_handle, read_size );
 if( MB_SUCCESS != rval ) // Wrong function for variable-length tags
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 // if (MB_TYPE_BIT == mbtype)
 // read_size = (read_size + 7) / 8; // Convert bits to bytes, plus 7 for ceiling
 
 if( hdf_read_type )
 { // If not opaque
 hsize_t hdf_size = H5Tget_size( hdf_read_type );
 if( hdf_size != (hsize_t)read_size ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 const int handles_per_tag = read_size / sizeof( EntityHandle );
 
 // Now read data values
 size_t chunk_size = bufferSize / read_size;
 try
 {
 ReadHDF5Dataset val_reader( ( tn + " values" ).c_str(), value_table, nativeParallel, mpiComm, false );
 val_reader.set_file_ids( offset_range, base_offset, chunk_size, hdf_read_type );
 dbgOut.printf( 3, "Reading sparse values for tag \"%s\" in %lu chunks\n", tn.c_str(),
 val_reader.get_read_count() );
 int nn = 0;
 size_t offset = 0;
 while( !val_reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of \"%s\" values\n", ++nn, tn.c_str() );
 size_t count;
 val_reader.read( dataBuffer, count );
 if( MB_TYPE_HANDLE == mbtype )
 {
 rval = convert_id_to_handle( (EntityHandle*)dataBuffer, count * handles_per_tag );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( !handle_vect.empty() )
 {
 rval = iFace->tag_set_data( tag_handle, &handle_vect[offset], count, dataBuffer );
 offset += count;
 }
 else
 {
 Range r;
 r.merge( handle_range.begin(), handle_range.begin() + count );
 handle_range.erase( handle_range.begin(), handle_range.begin() + count );
 rval = iFace->tag_set_data( tag_handle, r, dataBuffer );
 }
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_var_len_tag( Tag tag_handle,
 hid_t hdf_read_type,
 hid_t ent_table,
 hid_t val_table,
 hid_t off_table,
 long /*num_entities*/,
 long /*num_values*/ )
{
 CHECK_OPEN_HANDLES;
 
 ErrorCode rval;
 DataType mbtype;
 
 rval = iFace->tag_get_data_type( tag_handle, mbtype );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 // Can't do variable-length bit tags
 if( MB_TYPE_BIT == mbtype ) MB_CHK_ERR( MB_VARIABLE_DATA_LENGTH );
 
 // If here, MOAB tag must be variable-length
 int mbsize;
 if( MB_VARIABLE_DATA_LENGTH != iFace->tag_get_bytes( tag_handle, mbsize ) )
 {
 assert( false );MB_CHK_ERR( MB_VARIABLE_DATA_LENGTH );
 }
 
 int read_size;
 if( hdf_read_type )
 {
 hsize_t hdf_size = H5Tget_size( hdf_read_type );
 if( hdf_size < 1 ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 read_size = hdf_size;
 }
 else
 {
 // Opaque
 read_size = 1;
 }
 
 std::string tn( "<error>" );
 iFace->tag_get_name( tag_handle, tn );
 
 // Read entire ID table and for those file IDs corresponding
 // to entities that we have read from the file add both the
 // offset into the offset range and the handle into the handle
 // range. If handles are not ordered, switch to using a vector.
 const EntityHandle base_offset = 1; // Can't put zero in a Range
 std::vector< EntityHandle > handle_vect;
 Range handle_range, offset_range;
 rval = read_sparse_tag_indices( tn.c_str(), ent_table, base_offset, offset_range, handle_range, handle_vect );
 
 // This code only works if the id_table is an ordered list.
 // This assumption was also true for the previous iteration
 // of this code, but wasn't checked. MOAB's file writer
 // always writes an ordered list for id_table.
 if( !handle_vect.empty() )
 {
 MB_SET_ERR( MB_FAILURE, "Unordered file ids for variable length tag not supported" );
 }
 
 class VTReader : public ReadHDF5VarLen
 {
 Tag tagHandle;
 bool isHandle;
 size_t readSize;
 ReadHDF5* readHDF5;
 
 public:
 ErrorCode store_data( EntityHandle file_id, void* data, long count, bool )
 {
 ErrorCode rval1;
 if( isHandle )
 {
 if( readSize != sizeof( EntityHandle ) ) MB_CHK_SET_ERR( MB_FAILURE, "Invalid read size" );
 rval1 = readHDF5->convert_id_to_handle( (EntityHandle*)data, count );MB_CHK_ERR( rval1 );
 }
 int n = count;
 return readHDF5->moab()->tag_set_by_ptr( tagHandle, &file_id, 1, &data, &n );
 }
 VTReader( DebugOutput& debug_output,
 void* buffer,
 size_t buffer_size,
 Tag tag,
 bool is_handle_tag,
 size_t read_size1,
 ReadHDF5* owner )
 : ReadHDF5VarLen( debug_output, buffer, buffer_size ), tagHandle( tag ), isHandle( is_handle_tag ),
 readSize( read_size1 ), readHDF5( owner )
 {
 }
 };
 
 VTReader tool( dbgOut, dataBuffer, bufferSize, tag_handle, MB_TYPE_HANDLE == mbtype, read_size, this );
 try
 {
 // Read offsets into value table.
 std::vector< unsigned > counts;
 Range offsets;
 ReadHDF5Dataset off_reader( ( tn + " offsets" ).c_str(), off_table, nativeParallel, mpiComm, false );
 rval = tool.read_offsets( off_reader, offset_range, base_offset, base_offset, offsets, counts );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 // Read tag values
 Range empty;
 ReadHDF5Dataset val_reader( ( tn + " values" ).c_str(), val_table, nativeParallel, mpiComm, false );
 rval = tool.read_data( val_reader, offsets, base_offset, hdf_read_type, handle_range, counts, empty );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::convert_id_to_handle( EntityHandle* array, size_t size )
{
 convert_id_to_handle( array, size, idMap );
 return MB_SUCCESS;
}
 
void ReadHDF5::convert_id_to_handle( EntityHandle* array, size_t size, const RangeMap< long, EntityHandle >& id_map )
{
 for( EntityHandle* const end = array + size; array != end; ++array )
 *array = id_map.find( *array );
}
 
void ReadHDF5::convert_id_to_handle( EntityHandle* array,
 size_t size,
 size_t& new_size,
 const RangeMap< long, EntityHandle >& id_map )
{
 RangeMap< long, EntityHandle >::const_iterator it;
 new_size = 0;
 for( size_t i = 0; i < size; ++i )
 {
 it = id_map.lower_bound( array[i] );
 if( it != id_map.end() && it->begin <= (long)array[i] )
 array[new_size++] = it->value + ( array[i] - it->begin );
 }
}
 
void ReadHDF5::convert_range_to_handle( const EntityHandle* ranges,
 size_t num_ranges,
 const RangeMap< long, EntityHandle >& id_map,
 Range& merge )
{
 RangeMap< long, EntityHandle >::iterator it = id_map.begin();
 Range::iterator hint = merge.begin();
 for( size_t i = 0; i < num_ranges; ++i )
 {
 long id = ranges[2 * i];
 const long end = id + ranges[2 * i + 1];
 // We assume that 'ranges' is sorted, but check just in case it isn't.
 if( it == id_map.end() || it->begin > id ) it = id_map.begin();
 it = id_map.lower_bound( it, id_map.end(), id );
 if( it == id_map.end() ) continue;
 if( id < it->begin ) id = it->begin;
 while( id < end )
 {
 if( id < it->begin ) id = it->begin;
 const long off = id - it->begin;
 long count = std::min( it->count - off, end - id );
 // It is possible that this new subrange is starting after the end
 // It will result in negative count, which does not make sense
 // We are done with this range, go to the next one
 if( count <= 0 ) break;
 hint = merge.insert( hint, it->value + off, it->value + off + count - 1 );
 id += count;
 if( id < end )
 {
 if( ++it == id_map.end() ) break;
 if( it->begin > end ) break;
 }
 }
 }
}
 
ErrorCode ReadHDF5::convert_range_to_handle( const EntityHandle* array, size_t num_ranges, Range& range )
{
 convert_range_to_handle( array, num_ranges, idMap, range );
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::insert_in_id_map( const Range& file_ids, EntityHandle start_id )
{
 IDMap tmp_map;
 bool merge = !idMap.empty() && !file_ids.empty() && idMap.back().begin > (long)file_ids.front();
 IDMap& map = merge ? tmp_map : idMap;
 Range::const_pair_iterator p;
 for( p = file_ids.const_pair_begin(); p != file_ids.const_pair_end(); ++p )
 {
 size_t count = p->second - p->first + 1;
 if( !map.insert( p->first, start_id, count ).second ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 start_id += count;
 }
 if( merge && !idMap.merge( tmp_map ) ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::insert_in_id_map( long file_id, EntityHandle handle )
{
 if( !idMap.insert( file_id, handle, 1 ).second ) MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_qa( EntityHandle )
{
 CHECK_OPEN_HANDLES;
 
 mhdf_Status status;
 // std::vector<std::string> qa_list;
 
 int qa_len;
 char** qa = mhdf_readHistory( filePtr, &qa_len, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 // qa_list.resize(qa_len);
 for( int i = 0; i < qa_len; i++ )
 {
 // qa_list[i] = qa[i];
 free( qa[i] );
 }
 free( qa );
 
 /** FIX ME - how to put QA list on set?? */
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::store_file_ids( Tag tag )
{
 CHECK_OPEN_HANDLES;
 
 // typedef int tag_type;
 typedef long tag_type;
 // change it to be able to read much bigger files (long is 64 bits ...)
 
 tag_type* buffer = reinterpret_cast< tag_type* >( dataBuffer );
 const long buffer_size = bufferSize / sizeof( tag_type );
 for( IDMap::iterator i = idMap.begin(); i != idMap.end(); ++i )
 {
 IDMap::Range range = *i;
 
 // Make sure the values will fit in the tag type
 IDMap::key_type rv = range.begin + ( range.count - 1 );
 tag_type tv = (tag_type)rv;
 if( (IDMap::key_type)tv != rv )
 {
 assert( false );
 return MB_INDEX_OUT_OF_RANGE;
 }
 
 while( range.count )
 {
 long count = buffer_size < range.count ? buffer_size : range.count;
 
 Range handles;
 handles.insert( range.value, range.value + count - 1 );
 range.value += count;
 range.count -= count;
 for( long j = 0; j < count; ++j )
 buffer[j] = (tag_type)range.begin++;
 
 ErrorCode rval = iFace->tag_set_data( tag, handles, buffer );
 if( MB_SUCCESS != rval ) return rval;
 }
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::store_sets_file_ids()
{
 CHECK_OPEN_HANDLES;
 
 // create a tag that will not be saved, but it will be
 // used by visit plugin to match the sets and their file ids
 // it is the same type as the tag defined in ReadParallelcpp, for file id
 Tag setFileIdTag;
 long default_val = 0;
 ErrorCode rval = iFace->tag_get_handle( "__FILE_ID_FOR_SETS", sizeof( long ), MB_TYPE_OPAQUE, setFileIdTag,
 ( MB_TAG_DENSE | MB_TAG_CREAT ), &default_val );
 
 if( MB_SUCCESS != rval || 0 == setFileIdTag ) return rval;
 // typedef int tag_type;
 typedef long tag_type;
 // change it to be able to read much bigger files (long is 64 bits ...)
 
 tag_type* buffer = reinterpret_cast< tag_type* >( dataBuffer );
 const long buffer_size = bufferSize / sizeof( tag_type );
 for( IDMap::iterator i = idMap.begin(); i != idMap.end(); ++i )
 {
 IDMap::Range range = *i;
 EntityType htype = iFace->type_from_handle( range.value );
 if( MBENTITYSET != htype ) continue;
 // work only with entity sets
 // Make sure the values will fit in the tag type
 IDMap::key_type rv = range.begin + ( range.count - 1 );
 tag_type tv = (tag_type)rv;
 if( (IDMap::key_type)tv != rv )
 {
 assert( false );
 return MB_INDEX_OUT_OF_RANGE;
 }
 
 while( range.count )
 {
 long count = buffer_size < range.count ? buffer_size : range.count;
 
 Range handles;
 handles.insert( range.value, range.value + count - 1 );
 range.value += count;
 range.count -= count;
 for( long j = 0; j < count; ++j )
 buffer[j] = (tag_type)range.begin++;
 
 rval = iFace->tag_set_data( setFileIdTag, handles, buffer );
 if( MB_SUCCESS != rval ) return rval;
 }
 }
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_tag_values( const char* file_name,
 const char* tag_name,
 const FileOptions& opts,
 std::vector< int >& tag_values_out,
 const SubsetList* subset_list )
{
 ErrorCode rval;
 
 rval = set_up_read( file_name, opts );
 if( MB_SUCCESS != rval ) MB_SET_ERR( rval, "ReadHDF5 Failure" );
 
 int tag_index;
 rval = find_int_tag( tag_name, tag_index );
 if( MB_SUCCESS != rval )
 {
 clean_up_read( opts );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 if( subset_list )
 {
 Range file_ids;
 rval = get_subset_ids( subset_list->tag_list, subset_list->tag_list_length, file_ids );
 if( MB_SUCCESS != rval )
 {
 clean_up_read( opts );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 
 rval = read_tag_values_partial( tag_index, file_ids, tag_values_out );
 if( MB_SUCCESS != rval )
 {
 clean_up_read( opts );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 else
 {
 rval = read_tag_values_all( tag_index, tag_values_out );
 if( MB_SUCCESS != rval )
 {
 clean_up_read( opts );
 MB_SET_ERR( rval, "ReadHDF5 Failure" );
 }
 }
 
 return clean_up_read( opts );
}
 
ErrorCode ReadHDF5::read_tag_values_partial( int tag_index, const Range& file_ids, std::vector< int >& tag_values )
{
 CHECK_OPEN_HANDLES;
 
 mhdf_Status status;
 const mhdf_TagDesc& tag = fileInfo->tags[tag_index];
 long num_ent, num_val;
 size_t count;
 std::string tn( tag.name );
 
 // Read sparse values
 if( tag.have_sparse )
 {
 hid_t handles[3];
 mhdf_openSparseTagData( filePtr, tag.name, &num_ent, &num_val, handles, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 try
 {
 // Read all entity handles and fill 'offsets' with ranges of
 // offsets into the data table for entities that we want.
 Range offsets;
 long* buffer = reinterpret_cast< long* >( dataBuffer );
 const long buffer_size = bufferSize / sizeof( long );
 ReadHDF5Dataset ids( ( tn + " ids" ).c_str(), handles[0], nativeParallel, mpiComm );
 ids.set_all_file_ids( buffer_size, H5T_NATIVE_LONG );
 size_t offset = 0;
 dbgOut.printf( 3, "Reading sparse IDs for tag \"%s\" in %lu chunks\n", tag.name, ids.get_read_count() );
 int nn = 0;
 while( !ids.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of IDs for \"%s\"\n", ++nn, tag.name );
 ids.read( buffer, count );
 
 std::sort( buffer, buffer + count );
 Range::iterator ins = offsets.begin();
 Range::const_iterator i = file_ids.begin();
 for( size_t j = 0; j < count; ++j )
 {
 while( i != file_ids.end() && (long)*i < buffer[j] )
 ++i;
 if( i == file_ids.end() ) break;
 if( (long)*i == buffer[j] )
 {
 ins = offsets.insert( ins, j + offset, j + offset );
 }
 }
 
 offset += count;
 }
 
 tag_values.clear();
 tag_values.reserve( offsets.size() );
 const size_t data_buffer_size = bufferSize / sizeof( int );
 int* data_buffer = reinterpret_cast< int* >( dataBuffer );
 ReadHDF5Dataset vals( ( tn + " sparse vals" ).c_str(), handles[1], nativeParallel, mpiComm );
 vals.set_file_ids( offsets, 0, data_buffer_size, H5T_NATIVE_INT );
 dbgOut.printf( 3, "Reading sparse values for tag \"%s\" in %lu chunks\n", tag.name, vals.get_read_count() );
 nn = 0;
 // Should normally only have one read call, unless sparse nature
 // of file_ids caused reader to do something strange
 while( !vals.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of values for \"%s\"\n", ++nn, tag.name );
 vals.read( data_buffer, count );
 tag_values.insert( tag_values.end(), data_buffer, data_buffer + count );
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 }
 
 std::sort( tag_values.begin(), tag_values.end() );
 tag_values.erase( std::unique( tag_values.begin(), tag_values.end() ), tag_values.end() );
 
 // Read dense values
 std::vector< int > prev_data, curr_data;
 for( int i = 0; i < tag.num_dense_indices; ++i )
 {
 int grp = tag.dense_elem_indices[i];
 const char* gname = 0;
 mhdf_EntDesc* desc = 0;
 if( grp == -1 )
 {
 gname = mhdf_node_type_handle();
 desc = &fileInfo->nodes;
 }
 else if( grp == -2 )
 {
 gname = mhdf_set_type_handle();
 desc = &fileInfo->sets;
 }
 else
 {
 assert( grp >= 0 && grp < fileInfo->num_elem_desc );
 gname = fileInfo->elems[grp].handle;
 desc = &fileInfo->elems[grp].desc;
 }
 
 Range::iterator s = file_ids.lower_bound( (EntityHandle)( desc->start_id ) );
 Range::iterator e = Range::lower_bound( s, file_ids.end(), (EntityHandle)( desc->start_id ) + desc->count );
 Range subset;
 subset.merge( s, e );
 
 hid_t handle = mhdf_openDenseTagData( filePtr, tag.name, gname, &num_val, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 try
 {
 curr_data.clear();
 tag_values.reserve( subset.size() );
 const size_t data_buffer_size = bufferSize / sizeof( int );
 int* data_buffer = reinterpret_cast< int* >( dataBuffer );
 
 ReadHDF5Dataset reader( ( tn + " dense vals" ).c_str(), handle, nativeParallel, mpiComm );
 reader.set_file_ids( subset, desc->start_id, data_buffer_size, H5T_NATIVE_INT );
 dbgOut.printf( 3, "Reading dense data for tag \"%s\" and group \"%s\" in %lu chunks\n", tag.name,
 fileInfo->elems[grp].handle, reader.get_read_count() );
 int nn = 0;
 // Should normally only have one read call, unless sparse nature
 // of file_ids caused reader to do something strange
 while( !reader.done() )
 {
 dbgOut.printf( 3, "Reading chunk %d of \"%s\"/\"%s\"\n", ++nn, tag.name, fileInfo->elems[grp].handle );
 reader.read( data_buffer, count );
 curr_data.insert( curr_data.end(), data_buffer, data_buffer + count );
 }
 }
 catch( ReadHDF5Dataset::Exception )
 {
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 std::sort( curr_data.begin(), curr_data.end() );
 curr_data.erase( std::unique( curr_data.begin(), curr_data.end() ), curr_data.end() );
 prev_data.clear();
 tag_values.swap( prev_data );
 std::set_union( prev_data.begin(), prev_data.end(), curr_data.begin(), curr_data.end(),
 std::back_inserter( tag_values ) );
 }
 
 return MB_SUCCESS;
}
 
ErrorCode ReadHDF5::read_tag_values_all( int tag_index, std::vector< int >& tag_values )
{
 CHECK_OPEN_HANDLES;
 
 mhdf_Status status;
 const mhdf_TagDesc& tag = fileInfo->tags[tag_index];
 long junk, num_val;
 
 // Read sparse values
 if( tag.have_sparse )
 {
 hid_t handles[3];
 mhdf_openSparseTagData( filePtr, tag.name, &junk, &num_val, handles, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 mhdf_closeData( filePtr, handles[0], &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR_CONT( mhdf_message( &status ) );
 mhdf_closeData( filePtr, handles[1], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 hid_t file_type = H5Dget_type( handles[1] );
 tag_values.resize( num_val );
 mhdf_readTagValuesWithOpt( handles[1], 0, num_val, file_type, &tag_values[0], collIO, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR_CONT( mhdf_message( &status ) );
 H5Tclose( file_type );
 mhdf_closeData( filePtr, handles[1], &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 H5Tconvert( file_type, H5T_NATIVE_INT, num_val, &tag_values[0], 0, H5P_DEFAULT );
 H5Tclose( file_type );
 
 mhdf_closeData( filePtr, handles[1], &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 }
 
 std::sort( tag_values.begin(), tag_values.end() );
 tag_values.erase( std::unique( tag_values.begin(), tag_values.end() ), tag_values.end() );
 
 // Read dense values
 std::vector< int > prev_data, curr_data;
 for( int i = 0; i < tag.num_dense_indices; ++i )
 {
 int grp = tag.dense_elem_indices[i];
 const char* gname = 0;
 if( grp == -1 )
 gname = mhdf_node_type_handle();
 else if( grp == -2 )
 gname = mhdf_set_type_handle();
 else
 gname = fileInfo->elems[grp].handle;
 hid_t handle = mhdf_openDenseTagData( filePtr, tag.name, gname, &num_val, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 hid_t file_type = H5Dget_type( handle );
 curr_data.resize( num_val );
 mhdf_readTagValuesWithOpt( handle, 0, num_val, file_type, &curr_data[0], collIO, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR_CONT( mhdf_message( &status ) );
 H5Tclose( file_type );
 mhdf_closeData( filePtr, handle, &status );
 MB_SET_ERR( MB_FAILURE, "ReadHDF5 Failure" );
 }
 
 H5Tconvert( file_type, H5T_NATIVE_INT, num_val, &curr_data[0], 0, H5P_DEFAULT );
 H5Tclose( file_type );
 mhdf_closeData( filePtr, handle, &status );
 if( mhdf_isError( &status ) )
 {
 MB_SET_ERR( MB_FAILURE, mhdf_message( &status ) );
 }
 
 std::sort( curr_data.begin(), curr_data.end() );
 curr_data.erase( std::unique( curr_data.begin(), curr_data.end() ), curr_data.end() );
 
 prev_data.clear();
 tag_values.swap( prev_data );
 std::set_union( prev_data.begin(), prev_data.end(), curr_data.begin(), curr_data.end(),
 std::back_inserter( tag_values ) );
 }
 
 return MB_SUCCESS;
}
void ReadHDF5::print_times()
{
#ifdef MOAB_HAVE_MPI
 if( !myPcomm )
 {
 double recv[NUM_TIMES];
 MPI_Reduce( (void*)_times, recv, NUM_TIMES, MPI_DOUBLE, MPI_MAX, 0, myPcomm->proc_config().proc_comm() );
 for( int i = 0; i < NUM_TIMES; i++ )
 _times[i] = recv[i]; // just get the max from all of them
 }
 if( 0 == myPcomm->proc_config().proc_rank() )
 {
#endif
 
 std::cout << "ReadHDF5: " << _times[TOTAL_TIME] << std::endl
 << " get set meta " << _times[SET_META_TIME] << std::endl
 << " partial subsets " << _times[SUBSET_IDS_TIME] << std::endl
 << " partition time " << _times[GET_PARTITION_TIME] << std::endl
 << " get set ids " << _times[GET_SET_IDS_TIME] << std::endl
 << " set contents " << _times[GET_SET_CONTENTS_TIME] << std::endl
 << " polyhedra " << _times[GET_POLYHEDRA_TIME] << std::endl
 << " elements " << _times[GET_ELEMENTS_TIME] << std::endl
 << " nodes " << _times[GET_NODES_TIME] << std::endl
 << " node adjacency " << _times[GET_NODEADJ_TIME] << std::endl
 << " side elements " << _times[GET_SIDEELEM_TIME] << std::endl
 << " update connectivity " << _times[UPDATECONN_TIME] << std::endl
 << " adjacency " << _times[ADJACENCY_TIME] << std::endl
 << " delete non_adj " << _times[DELETE_NON_SIDEELEM_TIME] << std::endl
 << " recursive sets " << _times[READ_SET_IDS_RECURS_TIME] << std::endl
 << " find contain_sets " << _times[FIND_SETS_CONTAINING_TIME] << std::endl
 << " read sets " << _times[READ_SETS_TIME] << std::endl
 << " read tags " << _times[READ_TAGS_TIME] << std::endl
 << " store file ids " << _times[STORE_FILE_IDS_TIME] << std::endl
 << " read qa records " << _times[READ_QA_TIME] << std::endl;
 
#ifdef MOAB_HAVE_MPI
 }
#endif
}
 
} // namespace moab