CrystalRouterExample.cpp File Reference

Example demonstrating parallel communication using TupleList and Crystal Router. More...

#include "moab/MOABConfig.h"
#include "moab/TupleList.hpp"
#include "moab/ProgOptions.hpp"
#include "moab/ErrorHandler.hpp"
#include <ctime>
#include <iostream>
#include <sstream>

Include dependency graph for CrystalRouterExample.cpp:

Go to the source code of this file.

Functions
int	main (int argc, char **argv)

Variables
const char	BRIEF_DESC [] = "Example of gather scatter with tuple lists \n"
std::ostringstream	LONG_DESC

Detailed Description

Example demonstrating parallel communication using TupleList and Crystal Router.

This example shows how to:

• Use MOAB's TupleList for efficient data communication
• Perform sparse all-to-all communication using Crystal Router
• Handle parallel data exchange with varying communication patterns
• Verify received data integrity
• Measure communication performance

The Crystal Router is used for sparse communication patterns where each processor communicates with only a subset of other processors, avoiding the overhead of full all-to-all communication.

Author

MOAB Development Team

Date

2024

To run: mpiexec -np n CrystalRouterExample -r [reportrank] -t [num_tuples] -n [num_comms]

Parameters

argc	Number of command line arguments
argv	Command line arguments array

Returns

0 on success, 1 on failure

Definition in file CrystalRouterExample.cpp.

Function Documentation

main()

int main	(	int	argc,
		char **	argv
	)

Definition at line 78 of file CrystalRouterExample.cpp.

{
#ifdef MOAB_HAVE_MPI
    MPI_Init( &argc, &argv );
 
    // Initialize error handler, required for this example (not using a moab instance)
    MBErrorHandler_Init();
 
    ProcConfig pc( MPI_COMM_WORLD );
    int size = pc.proc_size();
    int rank = pc.proc_rank();
 
    // Start copy
    LONG_DESC << "This program does a gather scatter with a list of tuples. \n"
                 " It tries to see how much communication costs in terms of time and memory. \n"
              << "It starts with creating a list of tuples to be sent from each processor, \n to a "
                 "list of other processors.\n"
              << "The number of tuples and how many tasks to communicate to are controlled by "
                 "input parameters.\n"
              << "After communication, we verify locally if we received what we expected. \n";
    ProgOptions opts( LONG_DESC.str(), BRIEF_DESC );
 
    // How many procs communicate to current proc, on average (we will vary that too)
    int num_comms = 2;
    opts.addOpt< int >( "num_comms,n", "each task will send to about num_comms other tasks some tuples (default 2)",
                        &num_comms );
 
    int num_tuples = 4;
    opts.addOpt< int >( "num_tuples,t", "each task will send to some task about num_tuples tuples (default 4)",
                        &num_tuples );
 
    int reportrank = size + 1;
    opts.addOpt< int >( "reporting_rank,r",
                        "this rank will report the tuples sent and the tuples received; it could "
                        "be higher than num_procs, then no reporting",
                        &reportrank );
 
    opts.parseCommandLine( argc, argv );
 
    if( rank == reportrank || ( reportrank >= size && 0 == rank ) )
    {
        cout << " There are " << size << " tasks in example.\n";
        cout << " We will send groups of " << num_tuples << " from each task towards " << num_comms
             << " other tasks.\n";
    }
 
    // Send some data from proc i to i + n/2, also to i + n/2 + 1 modulo n, where n is num procs
 
    gs_data::crystal_data* cd = pc.crystal_router();
 
    long total_n_tuples = num_comms * num_tuples;
 
    // Vary the number of tasks to send to, and the number of tuples to send
    if( rank < size / 2 )
        num_comms--;
    else
        num_comms++;
 
    if( rank < size / 3 )
        num_tuples *= 2;
    else if( rank > size - size / 3 )
        num_tuples /= 2;
 
    TupleList tl;
    // At most num_tuples* num_comms to send
    // We do a preallocate with this; some tuples on some processors might need more memory, to be
    // able to grow locally; Some tasks might receive more tuples though, and in the process, some
    // might grow more than others. By doing these logP sends/receives, we do not grow local memory
    // too much.
    tl.initialize( 1, 1, 0, 1, num_tuples * num_comms );
    tl.enableWriteAccess();
    // Form num_tuples*num_comms tuples, send to various ranks
    unsigned int n = tl.get_n();
    for( int i = 0; i < num_comms; i++ )
    {
        int sendTo     = rank + i * size / 2 + 1;  // Spread out the send to, for a stress-like test
        sendTo         = sendTo % size;            //
        long intToSend = 1000 * rank + 100000 * sendTo;
        for( int j = 0; j < num_tuples; j++ )
        {
            n           = tl.get_n();
            tl.vi_wr[n] = sendTo;
            tl.vl_wr[n] = intToSend + j;
            tl.vr_wr[n] = 10000. * rank + j;
            tl.inc_n();
        }
    }
 
    if( rank == reportrank )
    {
        cout << "rank " << rank << "\n";
        tl.print( " before sending" );
    }
 
    clock_t tt = clock();
    // All communication happens here; no mpi calls for the user
    MB_CHK_SET_ERR( cd->gs_transfer( 1, tl, 0 ), "Error in tuple transfer" );
 
    double secs = 0;
    if( rank == reportrank || ( reportrank >= size && 0 == rank ) )
    {
        secs = ( clock() - tt ) / (double)CLOCKS_PER_SEC;
    }
    if( rank == reportrank )
    {
        cout << "rank " << rank << "\n";
        tl.print( " after transfer" );
    }
 
    // Check that all tuples received have the form 10000*rank + 100*from
    unsigned int received = tl.get_n();
    for( int i = 0; i < (int)received; i++ )
    {
        int from      = tl.vi_rd[i];
        long valrec   = tl.vl_rd[i];
        int remainder = valrec - 100000 * rank - 1000 * from;
        if( remainder < 0 || remainder >= num_tuples * 4 )
            cout << " error: tuple " << i << " received at proc rank " << rank << " from proc " << from << " has value "
                 << valrec << " remainder " << remainder << "\n";
    }
 
    if( rank == reportrank || ( reportrank >= size && 0 == rank ) )
    {
        cout << "communication of about " << total_n_tuples << " tuples/per proc took " << secs << " seconds" << "\n";
        tt = clock();
    }
 
    // Finalize error handler, required for this example (not using a moab instance)
    MBErrorHandler_Finalize();
 
    MPI_Finalize();
#else
    std::cout << " Build with MPI for this example to work\n";
#endif
 
    return 0;
}

References ProgOptions::addOpt(), BRIEF_DESC, moab::ProcConfig::crystal_router(), moab::TupleList::enableWriteAccess(), moab::TupleList::get_n(), moab::TupleList::inc_n(), moab::TupleList::initialize(), LONG_DESC, MB_CHK_SET_ERR, moab::MBErrorHandler_Finalize(), moab::MBErrorHandler_Init(), ProgOptions::parseCommandLine(), moab::TupleList::print(), moab::ProcConfig::proc_rank(), moab::ProcConfig::proc_size(), moab::TupleList::vi_rd, moab::TupleList::vi_wr, moab::TupleList::vl_rd, moab::TupleList::vl_wr, and moab::TupleList::vr_wr.

Variable Documentation

BRIEF_DESC

const char BRIEF_DESC[] = "Example of gather scatter with tuple lists \n"

Definition at line 72 of file CrystalRouterExample.cpp.

Referenced by main().

LONG_DESC

std::ostringstream LONG_DESC

Definition at line 73 of file CrystalRouterExample.cpp.

Referenced by main().