query_device_mpi.c File Reference

Include dependency graph for query_device_mpi.c:

Go to the source code of this file.

Macros
#define	MAX_LOCAL_RANKS   (512)
#define	MPI_CALL(call, err_handle)

Functions
static int	cmp_fn (const void *a, const void *b)
static int	print_cuda_affinity (MPI_Comm comm, void *handle)
int	main (int argc, char *argv[])

Macro Definition Documentation

MAX_LOCAL_RANKS

#define MAX_LOCAL_RANKS   (512)

Definition at line 7 of file query_device_mpi.c.

MPI_CALL

#define MPI_CALL	(		call,
			err_handle
	)

Value:

    do { \
    int _status = (call);               \
    if (_status == MPI_SUCCESS)         \
        break;                          \
    err_handle;                         \
} while(0)

Definition at line 9 of file query_device_mpi.c.

Function Documentation

cmp_fn()

static int cmp_fn ( const void *  a, const void *  b  )

static

Definition at line 17 of file query_device_mpi.c.

{
    return (*(unsigned long *)a - *(unsigned long *)b);
}

Here is the caller graph for this function:

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 91 of file query_device_mpi.c.

{
    int quiet = 0;
    quiet = tests_quiet(argc, argv);
 
    MPI_Init(&argc, &argv);
 
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
    int retval = PAPI_library_init(PAPI_VER_CURRENT);
    if (retval != PAPI_VER_CURRENT) {
        test_fail(__FILE__, __LINE__, "PAPI_library_init failed\n", retval);
    }
 
    if (!quiet && rank == 0) {
        printf("Testing systedect component with PAPI %d.%d.%d\n",
               PAPI_VERSION_MAJOR(PAPI_VERSION),
               PAPI_VERSION_MINOR(PAPI_VERSION),
               PAPI_VERSION_REVISION(PAPI_VERSION));
    }
 
    void *handle;
    int enum_modifier = PAPI_DEV_TYPE_ENUM__CUDA;
 
    while (PAPI_enum_dev_type(enum_modifier, &handle) == PAPI_OK) {
        if (!quiet) {
            print_cuda_affinity(MPI_COMM_WORLD, handle);
        }
    }
 
    MPI_Barrier(MPI_COMM_WORLD);
    if (rank == 0) {
        test_pass(__FILE__);
    }
 
    MPI_Finalize();
    PAPI_shutdown();
    return 0;
}

Here is the call graph for this function:

print_cuda_affinity()

static int print_cuda_affinity ( MPI_Comm  comm, void *  handle  )

static

Definition at line 23 of file query_device_mpi.c.

{
    int shm_comm_size;
    MPI_Comm shm_comm = MPI_COMM_NULL;
    const char *name;
    int dev_count;
    int rank, local_rank;
    int ranks[MAX_LOCAL_RANKS] = { 0 };
    unsigned long uid;
    unsigned long uids[MAX_LOCAL_RANKS] = { 0 };
 
    MPI_CALL(MPI_Comm_rank(comm, &rank), return _status);
    MPI_CALL(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED,
                                 rank, MPI_INFO_NULL, &shm_comm),
             return _status);
    MPI_CALL(MPI_Comm_size(shm_comm, &shm_comm_size), return _status);
    MPI_CALL(MPI_Comm_rank(shm_comm, &local_rank), return _status);
 
    PAPI_get_dev_type_attr(handle, PAPI_DEV_TYPE_ATTR__INT_COUNT, &dev_count);
    int i;
    for (i = 0; i < dev_count; ++i) {
        PAPI_get_dev_attr(handle, i, PAPI_DEV_ATTR__CUDA_CHAR_DEVICE_NAME, &name);
        PAPI_get_dev_attr(handle, i, PAPI_DEV_ATTR__CUDA_ULONG_UID, &uid);
 
        MPI_CALL(MPI_Alltoall(&rank, 1, MPI_INT, ranks, 1, MPI_INT,
                              shm_comm),
                 return _status);
        MPI_CALL(MPI_Alltoall(&uid, 1, MPI_UNSIGNED_LONG, uids, 1,
                              MPI_UNSIGNED_LONG, shm_comm),
                 return _status);
 
        unsigned long sorted_uids[MAX_LOCAL_RANKS] = { 0 };
        unsigned long uniq_sorted_uids[MAX_LOCAL_RANKS] = { 0 };
        memcpy(sorted_uids, uids, sizeof(unsigned long) * shm_comm_size);
        qsort(sorted_uids, shm_comm_size, sizeof(unsigned long), cmp_fn);
 
        if (local_rank == 0) {
            int j, uniq_uids = 0;
            unsigned long curr_uid = 0;
            for (j = 0; j < shm_comm_size; ++j) {
                if (sorted_uids[j] != curr_uid) {
                    curr_uid = sorted_uids[j];
                    uniq_sorted_uids[uniq_uids++] = curr_uid;
                }
            } 
 
            int k, l, list[MAX_LOCAL_RANKS] = { 0 };
            for (j = 0, l = 0; j < uniq_uids; ++j) {
                for (k = 0; k < shm_comm_size; ++k) {
                    if (uids[k] == uniq_sorted_uids[j]) {
                        list[l++] = ranks[k];
                    }
                }
 
                printf( "GPU-%i Affinity                       : Name: %s, UID: %lu, Ranks: [ ",
                        i, name, uniq_sorted_uids[j] );
                for (k = 0; k < l; ++k) {
                    printf( "%d ", list[k] );
                }
                printf( "]\n" );
            }
        }
    }
 
    MPI_CALL(MPI_Comm_free(&shm_comm), return _status);
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function: