From koop at mvapich.cse.ohio-state.edu Sun Nov 2 14:15:48 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Sun Nov 2 14:15:58 2008 Subject: [mvapich-commit] r3109 - mvapich/trunk/mpid/ch_hybrid Message-ID: <200811021915.mA2JFmU6015820@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-02 14:15:46 -0500 (Sun, 02 Nov 2008) New Revision: 3109 Modified: mvapich/trunk/mpid/ch_hybrid/mv_finalize.c Log: * Do not need to free cq separately Modified: mvapich/trunk/mpid/ch_hybrid/mv_finalize.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/mv_finalize.c 2008-10-30 15:29:03 UTC (rev 3108) +++ mvapich/trunk/mpid/ch_hybrid/mv_finalize.c 2008-11-02 19:15:46 UTC (rev 3109) @@ -151,10 +151,7 @@ ibv_close_device(mvdev.hca[i].context); } - free(mvdev.cq); free(mvdev.hca); - - free(mvdev.lids); free(mvdev.qpns); free(mvdev.connections); From koop at mvapich.cse.ohio-state.edu Sun Nov 2 14:44:32 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Sun Nov 2 14:44:43 2008 Subject: [mvapich-commit] r3110 - mvapich2/trunk/src/mpid/ch3/channels/mrail/src/gen2 Message-ID: <200811021944.mA2JiWa3015863@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-02 14:44:32 -0500 (Sun, 02 Nov 2008) New Revision: 3110 Modified: mvapich2/trunk/src/mpid/ch3/channels/mrail/src/gen2/ibv_param.c Log: * QDR can benefit from a 2K MTU Modified: mvapich2/trunk/src/mpid/ch3/channels/mrail/src/gen2/ibv_param.c =================================================================== --- mvapich2/trunk/src/mpid/ch3/channels/mrail/src/gen2/ibv_param.c 2008-11-02 19:15:46 UTC (rev 3109) +++ mvapich2/trunk/src/mpid/ch3/channels/mrail/src/gen2/ibv_param.c 2008-11-02 19:44:32 UTC (rev 3110) @@ -771,6 +771,8 @@ if (proc->hca_type == MLX_PCI_EX_DDR) { rdma_default_mtu = IBV_MTU_2048; + } else if(proc->hca_type == MLX_CX_QDR) { + rdma_default_mtu = IBV_MTU_2048; } else { rdma_default_mtu = IBV_MTU_1024; } From koop at mvapich.cse.ohio-state.edu Tue Nov 4 12:22:47 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Tue Nov 4 12:22:59 2008 Subject: [mvapich-commit] r3111 - mvapich/trunk/src/env Message-ID: <200811041722.mA4HMl7L001637@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-04 12:22:45 -0500 (Tue, 04 Nov 2008) New Revision: 3111 Modified: mvapich/trunk/src/env/initutil.c Log: * Increase the number of allowed nodes for shared memory broadcast to 4K Modified: mvapich/trunk/src/env/initutil.c =================================================================== --- mvapich/trunk/src/env/initutil.c 2008-11-02 19:44:32 UTC (rev 3110) +++ mvapich/trunk/src/env/initutil.c 2008-11-04 17:22:45 UTC (rev 3111) @@ -151,7 +151,7 @@ #endif #ifdef _SMP_ -#define DEFAULT_SHMEM_BCAST_LEADERS 1024 +#define DEFAULT_SHMEM_BCAST_LEADERS 4096 int enable_shmem_collectives = 1; int shmem_bcast_leaders = DEFAULT_SHMEM_BCAST_LEADERS; From koop at mvapich.cse.ohio-state.edu Tue Nov 4 12:25:46 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Tue Nov 4 12:25:57 2008 Subject: [mvapich-commit] r3112 - mvapich2/trunk/src/mpi/init Message-ID: <200811041725.mA4HPkcY001658@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-04 12:25:45 -0500 (Tue, 04 Nov 2008) New Revision: 3112 Modified: mvapich2/trunk/src/mpi/init/initthread.c Log: * Increase the number of allowed nodes for shared memory broadcast to 4K Modified: mvapich2/trunk/src/mpi/init/initthread.c =================================================================== --- mvapich2/trunk/src/mpi/init/initthread.c 2008-11-04 17:22:45 UTC (rev 3111) +++ mvapich2/trunk/src/mpi/init/initthread.c 2008-11-04 17:25:45 UTC (rev 3112) @@ -48,7 +48,7 @@ MPICH_ThreadInfo_t MPIR_ThreadInfo = { 0 }; #if defined(_OSU_MVAPICH_) -#define DEFAULT_SHMEM_BCAST_LEADERS 1024 +#define DEFAULT_SHMEM_BCAST_LEADERS 4096 #endif /* _OSU_MVAPICH_ */ /* These are initialized as null (avoids making these into common symbols). From perkinjo at mvapich.cse.ohio-state.edu Wed Nov 5 12:27:34 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Wed Nov 5 12:27:46 2008 Subject: [mvapich-commit] r3114 - mvapich2/trunk/maint Message-ID: <200811051727.mA5HRYft010793@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-05 12:27:32 -0500 (Wed, 05 Nov 2008) New Revision: 3114 Modified: mvapich2/trunk/maint/Version Log: Update Version to 1.2 proper. Modified: mvapich2/trunk/maint/Version =================================================================== --- mvapich2/trunk/maint/Version 2008-11-04 17:26:17 UTC (rev 3113) +++ mvapich2/trunk/maint/Version 2008-11-05 17:27:32 UTC (rev 3114) @@ -1 +1 @@ -1.2.0rc2 +1.2.0 From koop at mvapich.cse.ohio-state.edu Wed Nov 5 17:28:54 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Wed Nov 5 17:29:06 2008 Subject: [mvapich-commit] r3117 - in mvapich/trunk/mpid: ch_gen2 ch_hybrid ch_psm ch_smp Message-ID: <200811052228.mA5MSsH8011283@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-05 17:28:52 -0500 (Wed, 05 Nov 2008) New Revision: 3117 Added: mvapich/trunk/mpid/ch_gen2/calltrace.c mvapich/trunk/mpid/ch_gen2/objtrace.c mvapich/trunk/mpid/ch_hybrid/calltrace.c mvapich/trunk/mpid/ch_hybrid/objtrace.c mvapich/trunk/mpid/ch_psm/calltrace.c mvapich/trunk/mpid/ch_psm/objtrace.c mvapich/trunk/mpid/ch_smp/calltrace.c mvapich/trunk/mpid/ch_smp/objtrace.c Modified: mvapich/trunk/mpid/ch_gen2/Makefile.in mvapich/trunk/mpid/ch_gen2/viapriv.h mvapich/trunk/mpid/ch_hybrid/Makefile.in mvapich/trunk/mpid/ch_psm/Makefile.in mvapich/trunk/mpid/ch_smp/Makefile.in Log: * Add base support for object and call tracing Modified: mvapich/trunk/mpid/ch_gen2/Makefile.in =================================================================== --- mvapich/trunk/mpid/ch_gen2/Makefile.in 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_gen2/Makefile.in 2008-11-05 22:28:52 UTC (rev 3117) @@ -68,7 +68,7 @@ collutils.c intra_rdma_barrier.c \ mpid_mcast.c mcst_grp.c ibmcgrp.c \ crc32h.c avl.c mem_hooks.c viacoalesce.c shmem_coll.c\ - async_progress.c + async_progress.c calltrace.c objtrace.c VIAOBJECTS = viainit.o viasend.o viarecv.o viacheck.o \ viapriv.o viaparam.o viutil.o vbuf.o cm_user.o\ @@ -76,7 +76,8 @@ mpid_hsend.o mpid_hrecv.o mpid_pack.o cmnargs.o \ mpid_misc.o dreg.o mpid_smpi.o collutils.o intra_rdma_barrier.o \ mpid_mcast.o mcst_grp.o ibmcgrp.o crc32h.o avl.o \ - mem_hooks.o viacoalesce.o shmem_coll.o async_progress.o + mem_hooks.o viacoalesce.o shmem_coll.o async_progress.o calltrace.o \ + objtrace.o # default_all is the target used by the MPICH build. It can be optimized @@ -157,7 +158,7 @@ process/pmgr_client.h process/minidaemon.h mpid_misc.c viaparam.h viaconfig.h viadev.h \ process/common_pmgr_collective.h process/client_pmgr_collective.h \ cmnargs.c dreg.h mpid.h mpid_smpi.h mpid_smpi.c \ - process/mpirun_util.c + process/mpirun_util.c calltrace.c # # Files from the original ADI that we need, but didn't change, Added: mvapich/trunk/mpid/ch_gen2/calltrace.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/calltrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_gen2/calltrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,60 @@ + +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#ifndef FPRINTF +#define FPRINTF fprintf +#endif + +#if defined(NEEDS_STDLIB_PROTOTYPES) +#include +/* + Some gcc installations have out-of-date include files and need these + definitions to handle the "missing" prototypes. This is NOT + autodetected, but is provided and can be selected by using a switch + on the options line. + + These are from stdlib.h, stdio.h, and unistd.h + */ +extern int FPRINTF(FILE*,const char*,...); +extern int fflush(FILE *); +#endif + +#include "calltrace.h" + +/* Declarations */ +#ifdef DEBUG_TRACE +char *(TR_stack[TR_MAX_STACK]); +int TR_stack_sp = 0, TR_stack_debug = 0; + +void TR_stack_init( int flag ) +{ + TR_stack_debug = flag; +} + +/* Generate a stack trace */ +void TR_stack_print( + FILE *fp, + int dir ) +{ + int i; + + if (dir == 1) { + for (i=0; i=0; i--) { + FPRINTF( fp, "(%d) %s\n", i, TR_stack[i] ); + } + } +} + +#else +void TR_stack_init( int flag ) +{ +} +#endif Added: mvapich/trunk/mpid/ch_gen2/objtrace.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/objtrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_gen2/objtrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,23 @@ +#include "mpiimpl.h" + +#include + +FILE *MPIR_Ref_fp = 0; +int MPIR_Ref_flags = 0; + +void MPIR_Ref_init( + int flag, + char *filename) +{ + MPIR_Ref_flags = flag; + if (flag) { + if (filename) { + MPIR_Ref_fp = fopen( filename, "w" ); + if (!MPIR_Ref_fp) { + MPIR_Ref_flags = 0; + } + } + else + MPIR_Ref_fp = stdout; + } +} Modified: mvapich/trunk/mpid/ch_gen2/viapriv.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/viapriv.h 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_gen2/viapriv.h 2008-11-05 22:28:52 UTC (rev 3117) @@ -30,6 +30,7 @@ #include +#include "calltrace.h" #include "viaparam.h" #include "ibverbs_header.h" Modified: mvapich/trunk/mpid/ch_hybrid/Makefile.in =================================================================== --- mvapich/trunk/mpid/ch_hybrid/Makefile.in 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_hybrid/Makefile.in 2008-11-05 22:28:52 UTC (rev 3117) @@ -80,7 +80,7 @@ mv_send_xrc.c shmem_coll.c mv_init_rc_qp.c mv_rndv.c \ mv_rndv_rput.c mv_rndv_r3.c mv_rndv_rput_rel.c mv_rndv_rput_unrel.c \ mv_init_rc_fp.c mv_connect.c \ - mv_rpool.c mv_init_xrc_qp.c + mv_rpool.c mv_init_xrc_qp.c objtrace.c calltrace.c VIAOBJECTS = mv_init.o mv_finalize.o \ mv_send.o mv_recv.o mv_check.o mv_threads.o \ @@ -94,7 +94,7 @@ mv_send_xrc.o shmem_coll.o mv_init_rc_qp.o mv_rndv.o \ mv_rndv_rput.o mv_rndv_r3.o mv_rndv_rput_rel.o mv_rndv_rput_unrel.o \ mv_init_rc_fp.o mv_connect.o \ - mv_rpool.o mv_init_xrc_qp.o + mv_rpool.o mv_init_xrc_qp.o objtrace.o calltrace.o # default_all is the target used by the MPICH build. It can be optimized # to not to the ranlib that default does. Is this necessary on modern machines? Added: mvapich/trunk/mpid/ch_hybrid/calltrace.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/calltrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_hybrid/calltrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,60 @@ + +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#ifndef FPRINTF +#define FPRINTF fprintf +#endif + +#if defined(NEEDS_STDLIB_PROTOTYPES) +#include +/* + Some gcc installations have out-of-date include files and need these + definitions to handle the "missing" prototypes. This is NOT + autodetected, but is provided and can be selected by using a switch + on the options line. + + These are from stdlib.h, stdio.h, and unistd.h + */ +extern int FPRINTF(FILE*,const char*,...); +extern int fflush(FILE *); +#endif + +#include "calltrace.h" + +/* Declarations */ +#ifdef DEBUG_TRACE +char *(TR_stack[TR_MAX_STACK]); +int TR_stack_sp = 0, TR_stack_debug = 0; + +void TR_stack_init( int flag ) +{ + TR_stack_debug = flag; +} + +/* Generate a stack trace */ +void TR_stack_print( + FILE *fp, + int dir ) +{ + int i; + + if (dir == 1) { + for (i=0; i=0; i--) { + FPRINTF( fp, "(%d) %s\n", i, TR_stack[i] ); + } + } +} + +#else +void TR_stack_init( int flag ) +{ +} +#endif Added: mvapich/trunk/mpid/ch_hybrid/objtrace.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/objtrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_hybrid/objtrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,23 @@ +#include "mpiimpl.h" + +#include + +FILE *MPIR_Ref_fp = 0; +int MPIR_Ref_flags = 0; + +void MPIR_Ref_init( + int flag, + char *filename) +{ + MPIR_Ref_flags = flag; + if (flag) { + if (filename) { + MPIR_Ref_fp = fopen( filename, "w" ); + if (!MPIR_Ref_fp) { + MPIR_Ref_flags = 0; + } + } + else + MPIR_Ref_fp = stdout; + } +} Modified: mvapich/trunk/mpid/ch_psm/Makefile.in =================================================================== --- mvapich/trunk/mpid/ch_psm/Makefile.in 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_psm/Makefile.in 2008-11-05 22:28:52 UTC (rev 3117) @@ -63,13 +63,14 @@ psmpriv.c psmparam.c \ mpid_init.c mpid_recv.c mpid_send.c \ mpid_hsend.c mpid_hrecv.c mpid_pack.c \ - cmnargs.c mpid_misc.c shmem_coll.c mpid_smpi.c + cmnargs.c mpid_misc.c shmem_coll.c mpid_smpi.c objtrace.c calltrace.c VIAOBJECTS = psmprobe.o psmwait.o psminit.o psmsend.o psmrecv.o psmcheck.o \ psmpriv.o psmparam.o \ mpid_init.o mpid_send.o mpid_recv.o \ mpid_hsend.o mpid_hrecv.o mpid_pack.o cmnargs.o \ - mpid_misc.o shmem_coll.o mpid_smpi.o + mpid_misc.o shmem_coll.o mpid_smpi.o \ + objtrace.o calltrace.o # default_all is the target used by the MPICH build. It can be optimized # to not to the ranlib that default does. Is this necessary on modern machines? Added: mvapich/trunk/mpid/ch_psm/calltrace.c =================================================================== --- mvapich/trunk/mpid/ch_psm/calltrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_psm/calltrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,60 @@ + +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#ifndef FPRINTF +#define FPRINTF fprintf +#endif + +#if defined(NEEDS_STDLIB_PROTOTYPES) +#include +/* + Some gcc installations have out-of-date include files and need these + definitions to handle the "missing" prototypes. This is NOT + autodetected, but is provided and can be selected by using a switch + on the options line. + + These are from stdlib.h, stdio.h, and unistd.h + */ +extern int FPRINTF(FILE*,const char*,...); +extern int fflush(FILE *); +#endif + +#include "calltrace.h" + +/* Declarations */ +#ifdef DEBUG_TRACE +char *(TR_stack[TR_MAX_STACK]); +int TR_stack_sp = 0, TR_stack_debug = 0; + +void TR_stack_init( int flag ) +{ + TR_stack_debug = flag; +} + +/* Generate a stack trace */ +void TR_stack_print( + FILE *fp, + int dir ) +{ + int i; + + if (dir == 1) { + for (i=0; i=0; i--) { + FPRINTF( fp, "(%d) %s\n", i, TR_stack[i] ); + } + } +} + +#else +void TR_stack_init( int flag ) +{ +} +#endif Added: mvapich/trunk/mpid/ch_psm/objtrace.c =================================================================== --- mvapich/trunk/mpid/ch_psm/objtrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_psm/objtrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,23 @@ +#include "mpiimpl.h" + +#include + +FILE *MPIR_Ref_fp = 0; +int MPIR_Ref_flags = 0; + +void MPIR_Ref_init( + int flag, + char *filename) +{ + MPIR_Ref_flags = flag; + if (flag) { + if (filename) { + MPIR_Ref_fp = fopen( filename, "w" ); + if (!MPIR_Ref_fp) { + MPIR_Ref_flags = 0; + } + } + else + MPIR_Ref_fp = stdout; + } +} Modified: mvapich/trunk/mpid/ch_smp/Makefile.in =================================================================== --- mvapich/trunk/mpid/ch_smp/Makefile.in 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_smp/Makefile.in 2008-11-05 22:28:52 UTC (rev 3117) @@ -64,14 +64,14 @@ mpid_init.c mpid_recv.c mpid_send.c \ mpid_hsend.c mpid_hrecv.c mpid_pack.c \ cmnargs.c mpid_misc.c mpid_smpi.c \ - shmem_coll.c + shmem_coll.c objtrace.c calltrace.c VIAOBJECTS = viainit.o viasend.o viarecv.o viacheck.o \ viaparam.o viutil.o \ mpid_init.o mpid_send.o mpid_recv.o \ mpid_hsend.o mpid_hrecv.o mpid_pack.o cmnargs.o\ mpid_misc.o mpid_smpi.o \ - shmem_coll.o + shmem_coll.o objtrace.o calltrace.o # default_all is the target used by the MPICH build. It can be optimized # to not to the ranlib that default does. Is this necessary on modern machines? Added: mvapich/trunk/mpid/ch_smp/calltrace.c =================================================================== --- mvapich/trunk/mpid/ch_smp/calltrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_smp/calltrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,60 @@ + +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#ifndef FPRINTF +#define FPRINTF fprintf +#endif + +#if defined(NEEDS_STDLIB_PROTOTYPES) +#include +/* + Some gcc installations have out-of-date include files and need these + definitions to handle the "missing" prototypes. This is NOT + autodetected, but is provided and can be selected by using a switch + on the options line. + + These are from stdlib.h, stdio.h, and unistd.h + */ +extern int FPRINTF(FILE*,const char*,...); +extern int fflush(FILE *); +#endif + +#include "calltrace.h" + +/* Declarations */ +#ifdef DEBUG_TRACE +char *(TR_stack[TR_MAX_STACK]); +int TR_stack_sp = 0, TR_stack_debug = 0; + +void TR_stack_init( int flag ) +{ + TR_stack_debug = flag; +} + +/* Generate a stack trace */ +void TR_stack_print( + FILE *fp, + int dir ) +{ + int i; + + if (dir == 1) { + for (i=0; i=0; i--) { + FPRINTF( fp, "(%d) %s\n", i, TR_stack[i] ); + } + } +} + +#else +void TR_stack_init( int flag ) +{ +} +#endif Added: mvapich/trunk/mpid/ch_smp/objtrace.c =================================================================== --- mvapich/trunk/mpid/ch_smp/objtrace.c 2008-11-05 21:30:51 UTC (rev 3116) +++ mvapich/trunk/mpid/ch_smp/objtrace.c 2008-11-05 22:28:52 UTC (rev 3117) @@ -0,0 +1,23 @@ +#include "mpiimpl.h" + +#include + +FILE *MPIR_Ref_fp = 0; +int MPIR_Ref_flags = 0; + +void MPIR_Ref_init( + int flag, + char *filename) +{ + MPIR_Ref_flags = flag; + if (flag) { + if (filename) { + MPIR_Ref_fp = fopen( filename, "w" ); + if (!MPIR_Ref_fp) { + MPIR_Ref_flags = 0; + } + } + else + MPIR_Ref_fp = stdout; + } +} From chail at mvapich.cse.ohio-state.edu Thu Nov 6 15:46:53 2008 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Thu Nov 6 15:47:04 2008 Subject: [mvapich-commit] r3118 - mvapich2/trunk Message-ID: <200811062046.mA6KkrKZ020522@mvapich.cse.ohio-state.edu> Author: chail Date: 2008-11-06 15:46:53 -0500 (Thu, 06 Nov 2008) New Revision: 3118 Modified: mvapich2/trunk/CHANGELOG Log: Adding changelog content. Modified: mvapich2/trunk/CHANGELOG =================================================================== --- mvapich2/trunk/CHANGELOG 2008-11-05 22:28:52 UTC (rev 3117) +++ mvapich2/trunk/CHANGELOG 2008-11-06 20:46:53 UTC (rev 3118) @@ -3,6 +3,24 @@ This file briefly describes the changes to the MVAPICH2 software package. The logs are arranged in the "most recent first" order. +MVAPICH2-1.2 (11/06/2008) + +* Ignore the last bit of the pkey and remove the pkey_ix option since the index + can be different on different machines. Thanks for Pasha@Mellanox for the patch. + +* Use /dev/shm on Linux as the default temporary file path for shared memory + communication. Thanks for Doug Johnson@OSC for the patch. + +* MV2_DEFAULT_MAX_WQE has been replaced with MV2_DEFAULT_MAX_SEND_WQE and + MV2_DEFAULT_MAX_RECV_WQE for send and recv wqes, respectively. + +* Fix compilation warnings. + +* Fix data types for memory allocations. Thanks for Dr. Bill Barth from TACC + for the patches. + +* Fix a bug when MV2_NUM_HCAS is larger than the number of active HCAs. + MVAPICH2-1.2-RC2 (08/20/2008) * Following bugs are fixed in RC2 From perkinjo at mvapich.cse.ohio-state.edu Thu Nov 6 17:13:11 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Thu Nov 6 17:13:22 2008 Subject: [mvapich-commit] r3119 - mvapich2/trunk Message-ID: <200811062213.mA6MDBRh020700@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-06 17:13:10 -0500 (Thu, 06 Nov 2008) New Revision: 3119 Modified: mvapich2/trunk/CHANGELOG Log: Update changelog with changes since rc2 release. Modified: mvapich2/trunk/CHANGELOG =================================================================== --- mvapich2/trunk/CHANGELOG 2008-11-06 20:46:53 UTC (rev 3118) +++ mvapich2/trunk/CHANGELOG 2008-11-06 22:13:10 UTC (rev 3119) @@ -5,22 +5,47 @@ MVAPICH2-1.2 (11/06/2008) -* Ignore the last bit of the pkey and remove the pkey_ix option since the index - can be different on different machines. Thanks for Pasha@Mellanox for the patch. +* Bugs fixed since MVAPICH2-1.2-rc2 -* Use /dev/shm on Linux as the default temporary file path for shared memory - communication. Thanks for Doug Johnson@OSC for the patch. + - Ignore the last bit of the pkey and remove the pkey_ix option since the + index can be different on different machines. Thanks for Pasha@Mellanox for + the patch. -* MV2_DEFAULT_MAX_WQE has been replaced with MV2_DEFAULT_MAX_SEND_WQE and - MV2_DEFAULT_MAX_RECV_WQE for send and recv wqes, respectively. + - Fix data types for memory allocations. Thanks for Dr. Bill Barth from TACC + for the patches. -* Fix compilation warnings. + - Fix a bug when MV2_NUM_HCAS is larger than the number of active HCAs. -* Fix data types for memory allocations. Thanks for Dr. Bill Barth from TACC - for the patches. + - Allow builds on architectures for which tuning parameters do not exist. -* Fix a bug when MV2_NUM_HCAS is larger than the number of active HCAs. +* Changes related to the mpirun_rsh framework + - Always build and install mpirun_rsh in addition to the process manager(s) + selected through the --with-pm mechanism. + + - Cleaner job abort handling + + - Ability to detect the path to mpispawn if the Linux proc filesystem is + available. + + - Added Totalview debugger support + + - Stdin is only available to rank 0. Other ranks get /dev/null. + +* Other miscellaneous changes + + - Add sequence numbers for RPUT and RGET finish packets. + + - Increase the number of allowed nodes for shared memory broadcast to 4K. + + - Use /dev/shm on Linux as the default temporary file path for shared memory + communication. Thanks for Doug Johnson@OSC for the patch. + + - MV2_DEFAULT_MAX_WQE has been replaced with MV2_DEFAULT_MAX_SEND_WQE and + MV2_DEFAULT_MAX_RECV_WQE for send and recv wqes, respectively. + + - Fix compilation warnings. + MVAPICH2-1.2-RC2 (08/20/2008) * Following bugs are fixed in RC2 @@ -39,7 +64,8 @@ - Improve mvapich2-1.2 bandwidth to the same level of mvapich2-1.0.3. - - An error handling patch for uDAPL interface. Thanks for Nilesh Awate for the patch. + - An error handling patch for uDAPL interface. Thanks for Nilesh Awate for + the patch. - Explicitly set some of the EP attributes when on demand connection is used in uDAPL interface. From perkinjo at mvapich.cse.ohio-state.edu Thu Nov 6 17:46:36 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Thu Nov 6 17:46:47 2008 Subject: [mvapich-commit] r3120 - mvapich2/branches Message-ID: <200811062246.mA6Mkaau020778@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-06 17:46:35 -0500 (Thu, 06 Nov 2008) New Revision: 3120 Added: mvapich2/branches/1.2/ Log: Create mvapich2-1.2 branch. Copied: mvapich2/branches/1.2 (from rev 3119, mvapich2/trunk) From perkinjo at mvapich.cse.ohio-state.edu Thu Nov 6 17:47:31 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Thu Nov 6 17:47:42 2008 Subject: [mvapich-commit] r3121 - mvapich2/tags Message-ID: <200811062247.mA6MlVdY020796@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-06 17:47:30 -0500 (Thu, 06 Nov 2008) New Revision: 3121 Added: mvapich2/tags/1.2/ Log: Create mvapich2-1.2 release tag. Copied: mvapich2/tags/1.2 (from rev 3120, mvapich2/branches/1.2) From gangadha at mvapich.cse.ohio-state.edu Sat Nov 8 02:30:08 2008 From: gangadha at mvapich.cse.ohio-state.edu (gangadha@mvapich.cse.ohio-state.edu) Date: Sat Nov 8 02:30:20 2008 Subject: [mvapich-commit] r3122 - mvapich/trunk/mpid/ch_psm Message-ID: <200811080730.mA87U8AP011053@mvapich.cse.ohio-state.edu> Author: gangadha Date: 2008-11-08 02:30:06 -0500 (Sat, 08 Nov 2008) New Revision: 3122 Modified: mvapich/trunk/mpid/ch_psm/psminit.c Log: processes array not allocated in psminit. added allocation code. Modified: mvapich/trunk/mpid/ch_psm/psminit.c =================================================================== --- mvapich/trunk/mpid/ch_psm/psminit.c 2008-11-06 22:47:30 UTC (rev 3121) +++ mvapich/trunk/mpid/ch_psm/psminit.c 2008-11-08 07:30:06 UTC (rev 3122) @@ -180,6 +180,7 @@ psmdev.pids = (int *) malloc(psmdev.np * sizeof(int)); psmdev.pids[psmdev.me] = (int) getpid(); + psmdev.processes = (char **)malloc(psmdev.np * sizeof(char *)); pmgr_allgather(&(psmdev.pids[psmdev.me]), sizeof(int), psmdev.pids); From chail at mvapich.cse.ohio-state.edu Sat Nov 8 19:13:34 2008 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Sat Nov 8 19:13:45 2008 Subject: [mvapich-commit] r3123 - mvapich2/trunk/src/mpid/ch3/channels/mrail/src/rdma Message-ID: <200811090013.mA90DY65013149@mvapich.cse.ohio-state.edu> Author: chail Date: 2008-11-08 19:13:33 -0500 (Sat, 08 Nov 2008) New Revision: 3123 Modified: mvapich2/trunk/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c Log: Fix a bug in shared memory communication. Modified: mvapich2/trunk/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c =================================================================== --- mvapich2/trunk/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c 2008-11-08 07:30:06 UTC (rev 3122) +++ mvapich2/trunk/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c 2008-11-09 00:13:33 UTC (rev 3123) @@ -333,6 +333,7 @@ MPID_Request_release(send_req); } + vc->smp.send_current_pkt_type = SMP_RNDV_MSG; DEBUG_PRINT("r3 sent req is %p\n", sreq); if (MPIDI_CH3I_SMP_SendQ_empty(vc)) { @@ -341,10 +342,18 @@ sreq->dev.iov_count, sreq->dev.iov_offset, sreq->dev.iov[0].MPID_IOV_LEN); - mpi_errno = MPIDI_CH3I_SMP_writev_rndv_data(vc, + if (vc->smp.send_current_pkt_type == SMP_RNDV_MSG) { + mpi_errno = MPIDI_CH3I_SMP_writev_rndv_data(vc, &sreq->dev.iov[sreq->dev.iov_offset], sreq->dev.iov_count - sreq->dev.iov_offset, &nb); + } else { + MPIU_Assert(vc->smp.send_current_pkt_type == SMP_RNDV_MSG_CONT); + MPIDI_CH3I_SMP_writev_rndv_data_cont(vc, + &sreq->dev.iov[sreq->dev.iov_offset], + sreq->dev.iov_count - sreq->dev.iov_offset, + &nb); + } if (MPI_SUCCESS != mpi_errno) { vc->ch.state = MPIDI_CH3I_VC_STATE_FAILED; From chail at mvapich.cse.ohio-state.edu Sat Nov 8 19:31:33 2008 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Sat Nov 8 19:31:44 2008 Subject: [mvapich-commit] r3125 - mvapich2/branches/1.2/src/mpid/ch3/channels/mrail/src/rdma Message-ID: <200811090031.mA90VX9b013202@mvapich.cse.ohio-state.edu> Author: chail Date: 2008-11-08 19:31:31 -0500 (Sat, 08 Nov 2008) New Revision: 3125 Modified: mvapich2/branches/1.2/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c Log: Fix a bug in shared memory communication. Modified: mvapich2/branches/1.2/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c =================================================================== --- mvapich2/branches/1.2/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c 2008-11-09 00:22:23 UTC (rev 3124) +++ mvapich2/branches/1.2/src/mpid/ch3/channels/mrail/src/rdma/ch3_rndvtransfer.c 2008-11-09 00:31:31 UTC (rev 3125) @@ -333,6 +333,7 @@ MPID_Request_release(send_req); } + vc->smp.send_current_pkt_type = SMP_RNDV_MSG; DEBUG_PRINT("r3 sent req is %p\n", sreq); if (MPIDI_CH3I_SMP_SendQ_empty(vc)) { @@ -341,10 +342,18 @@ sreq->dev.iov_count, sreq->dev.iov_offset, sreq->dev.iov[0].MPID_IOV_LEN); - mpi_errno = MPIDI_CH3I_SMP_writev_rndv_data(vc, + if (vc->smp.send_current_pkt_type == SMP_RNDV_MSG) { + mpi_errno = MPIDI_CH3I_SMP_writev_rndv_data(vc, &sreq->dev.iov[sreq->dev.iov_offset], sreq->dev.iov_count - sreq->dev.iov_offset, &nb); + } else { + MPIU_Assert(vc->smp.send_current_pkt_type == SMP_RNDV_MSG_CONT); + MPIDI_CH3I_SMP_writev_rndv_data_cont(vc, + &sreq->dev.iov[sreq->dev.iov_offset], + sreq->dev.iov_count - sreq->dev.iov_offset, + &nb); + } if (MPI_SUCCESS != mpi_errno) { vc->ch.state = MPIDI_CH3I_VC_STATE_FAILED; From gangadha at mvapich.cse.ohio-state.edu Sun Nov 9 12:23:11 2008 From: gangadha at mvapich.cse.ohio-state.edu (gangadha@mvapich.cse.ohio-state.edu) Date: Sun Nov 9 12:23:24 2008 Subject: [mvapich-commit] r3126 - mvapich/trunk/mpid/ch_psm Message-ID: <200811091723.mA9HNBCD000393@mvapich.cse.ohio-state.edu> Author: gangadha Date: 2008-11-09 12:23:10 -0500 (Sun, 09 Nov 2008) New Revision: 3126 Modified: mvapich/trunk/mpid/ch_psm/psminit.c Log: changes to r3122 as suggested by Jaidev. Modified: mvapich/trunk/mpid/ch_psm/psminit.c =================================================================== --- mvapich/trunk/mpid/ch_psm/psminit.c 2008-11-09 00:31:31 UTC (rev 3125) +++ mvapich/trunk/mpid/ch_psm/psminit.c 2008-11-09 17:23:10 UTC (rev 3126) @@ -145,6 +145,7 @@ psm_error_t *errors; char temp_str[100]; char temp_str1[100]; + char *buf; int j; int my_local_id=0, num_local_nodes=0; @@ -180,7 +181,7 @@ psmdev.pids = (int *) malloc(psmdev.np * sizeof(int)); psmdev.pids[psmdev.me] = (int) getpid(); - psmdev.processes = (char **)malloc(psmdev.np * sizeof(char *)); + pmgr_allgatherstr(psmdev.my_name, &psmdev.processes, &buf); pmgr_allgather(&(psmdev.pids[psmdev.me]), sizeof(int), psmdev.pids); From subramon at mvapich.cse.ohio-state.edu Mon Nov 10 16:36:22 2008 From: subramon at mvapich.cse.ohio-state.edu (subramon@mvapich.cse.ohio-state.edu) Date: Mon Nov 10 16:36:35 2008 Subject: [mvapich-commit] r3127 - in mvapich/trunk/src: coll env Message-ID: <200811102136.mAALaMd0015869@mvapich.cse.ohio-state.edu> Author: subramon Date: 2008-11-10 16:36:18 -0500 (Mon, 10 Nov 2008) New Revision: 3127 Modified: mvapich/trunk/src/coll/intra_fns_new.c mvapich/trunk/src/env/initutil.c Log: Change to fix compilation error seen with blocking combination Modified: mvapich/trunk/src/coll/intra_fns_new.c =================================================================== --- mvapich/trunk/src/coll/intra_fns_new.c 2008-11-09 17:23:10 UTC (rev 3126) +++ mvapich/trunk/src/coll/intra_fns_new.c 2008-11-10 21:36:18 UTC (rev 3127) @@ -71,11 +71,12 @@ int shmem_coll_reduce_threshold = (1<<10); int shmem_coll_allreduce_threshold = (1<<15); int shmem_coll_bcast_threshold = (1<<23); -long int allgather_large_msg_threshold = MPIR_ALLGATHER_LONG_MSG; -long int allgather_small_msg_threshold = MPIR_ALLGATHER_SHORT_MSG; #endif int disable_shmem_bcast =0; #endif +/* Tuning thresholds for allgather */ +long int allgather_large_msg_threshold = MPIR_ALLGATHER_LONG_MSG; +long int allgather_small_msg_threshold = MPIR_ALLGATHER_SHORT_MSG; /* Knomial Bcast */ int bcast_knomial_degree = 4; Modified: mvapich/trunk/src/env/initutil.c =================================================================== --- mvapich/trunk/src/env/initutil.c 2008-11-09 17:23:10 UTC (rev 3126) +++ mvapich/trunk/src/env/initutil.c 2008-11-10 21:36:18 UTC (rev 3127) @@ -166,13 +166,13 @@ extern int shmem_coll_reduce_threshold; extern int shmem_coll_allreduce_threshold; extern int shmem_coll_bcast_threshold; -extern long int allgather_large_msg_threshold; -extern long int allgather_small_msg_threshold; extern void clear_2level_comm (struct MPIR_COMMUNICATOR*); extern void create_2level_comm (struct MPIR_COMMUNICATOR*, int, int); #endif #endif extern int bcast_knomial_degree; +extern long int allgather_large_msg_threshold; +extern long int allgather_small_msg_threshold; int viadev_use_allgather_new =1; int enable_rdma_collectives = 0; /* @@ -300,6 +300,7 @@ printf("Shmem_coll_max_msg_size should be greater than the thresholds\n"); exit(0); } +#endif if ((value = getenv("MV_ALLGATHER_LARGE_MSG_THRESHOLD")) != NULL ) { if (atol(value) >= 0){ allgather_large_msg_threshold = atol(value); @@ -310,7 +311,6 @@ allgather_small_msg_threshold = atol(value); } } -#endif if ((value = getenv("VIADEV_BCAST_KNOMIAL")) != NULL){ bcast_knomial_degree = atoi(value); if(bcast_knomial_degree<2) From chail at mvapich.cse.ohio-state.edu Mon Nov 10 21:07:21 2008 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Mon Nov 10 21:07:34 2008 Subject: [mvapich-commit] r3128 - mvapich/trunk/src/coll Message-ID: <200811110207.mAB27LPt016351@mvapich.cse.ohio-state.edu> Author: chail Date: 2008-11-10 21:07:19 -0500 (Mon, 10 Nov 2008) New Revision: 3128 Modified: mvapich/trunk/src/coll/intra_fns_new.c Log: Fix a bug for intel test c/collective/error/MPI_Allreduce_err1. Modified: mvapich/trunk/src/coll/intra_fns_new.c =================================================================== --- mvapich/trunk/src/coll/intra_fns_new.c 2008-11-10 21:36:18 UTC (rev 3127) +++ mvapich/trunk/src/coll/intra_fns_new.c 2008-11-11 02:07:19 UTC (rev 3128) @@ -5167,10 +5167,13 @@ uop = op_ptr->op; MPIR_Op_errno = MPI_SUCCESS; - int test = 0; - (*uop)(sendbuf, recvbuf, &test, &datatype->self); - if (MPIR_Op_errno != MPI_SUCCESS) return MPIR_Op_errno; + if (op >= MPI_MAX && op <= MPI_MAXLOC) { + int test = 0; + (*uop)(sendbuf, recvbuf, &test, &datatype->self); + if (MPIR_Op_errno != MPI_SUCCESS) return MPIR_Op_errno; + } + MPI_Type_extent(datatype->self, &extent); MPI_Type_lb( datatype->self, &lb ); @@ -5952,6 +5955,12 @@ MPIR_Op_errno = MPI_SUCCESS; + if (op >= MPI_MAX && op <= MPI_MAXLOC) { + int test = 0; + (*uop)(sendbuf, recvbuf, &test, &datatype->self); + if (MPIR_Op_errno != MPI_SUCCESS) return MPIR_Op_errno; + } + stride = count*extent; if ((comm_ptr->shmem_coll_ok == 1) && From perkinjo at mvapich.cse.ohio-state.edu Tue Nov 11 12:45:06 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Tue Nov 11 12:45:19 2008 Subject: [mvapich-commit] r3129 - mvapich2/trunk Message-ID: <200811111745.mABHj63u030684@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-11 12:45:04 -0500 (Tue, 11 Nov 2008) New Revision: 3129 Modified: mvapich2/trunk/CHANGELOG Log: Update CHANGELOG for mvapich2-1.2p1. Modified: mvapich2/trunk/CHANGELOG =================================================================== --- mvapich2/trunk/CHANGELOG 2008-11-11 02:07:19 UTC (rev 3128) +++ mvapich2/trunk/CHANGELOG 2008-11-11 17:45:04 UTC (rev 3129) @@ -3,6 +3,12 @@ This file briefly describes the changes to the MVAPICH2 software package. The logs are arranged in the "most recent first" order. +MVAPICH2-1.2p1 (11/11/2008) + +* Changes since MVAPICH2-1.2 + + - Fix shared-memory communication issue for AMD Barcelona systems. + MVAPICH2-1.2 (11/06/2008) * Bugs fixed since MVAPICH2-1.2-rc2 From perkinjo at mvapich.cse.ohio-state.edu Tue Nov 11 13:26:59 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Tue Nov 11 13:27:11 2008 Subject: [mvapich-commit] r3130 - mvapich2/trunk/maint Message-ID: <200811111826.mABIQxgx030838@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-11 13:26:57 -0500 (Tue, 11 Nov 2008) New Revision: 3130 Modified: mvapich2/trunk/maint/Version Log: Update version string. Modified: mvapich2/trunk/maint/Version =================================================================== --- mvapich2/trunk/maint/Version 2008-11-11 17:45:04 UTC (rev 3129) +++ mvapich2/trunk/maint/Version 2008-11-11 18:26:57 UTC (rev 3130) @@ -1 +1 @@ -1.2.0 +1.2.0p1 From perkinjo at mvapich.cse.ohio-state.edu Tue Nov 11 15:39:46 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Tue Nov 11 15:39:58 2008 Subject: [mvapich-commit] r3131 - mvapich2/tags Message-ID: <200811112039.mABKdkMe031128@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-11 15:39:44 -0500 (Tue, 11 Nov 2008) New Revision: 3131 Added: mvapich2/tags/1.2p1/ Log: Create tag for mvapich2-1.2p1. Copied: mvapich2/tags/1.2p1 (from rev 3130, mvapich2/trunk) From perkinjo at mvapich.cse.ohio-state.edu Tue Nov 11 17:23:59 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Tue Nov 11 17:24:12 2008 Subject: [mvapich-commit] r3132 - in mvapich2/branches/1.2: . maint Message-ID: <200811112223.mABMNxpp031414@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-11 17:23:57 -0500 (Tue, 11 Nov 2008) New Revision: 3132 Modified: mvapich2/branches/1.2/CHANGELOG mvapich2/branches/1.2/maint/Version Log: Update mvapich2 1.2 branch CHANGELOG and Version files to 1.2p1. Modified: mvapich2/branches/1.2/CHANGELOG =================================================================== --- mvapich2/branches/1.2/CHANGELOG 2008-11-11 20:39:44 UTC (rev 3131) +++ mvapich2/branches/1.2/CHANGELOG 2008-11-11 22:23:57 UTC (rev 3132) @@ -3,6 +3,12 @@ This file briefly describes the changes to the MVAPICH2 software package. The logs are arranged in the "most recent first" order. +MVAPICH2-1.2p1 (11/11/2008) + +* Changes since MVAPICH2-1.2 + + - Fix shared-memory communication issue for AMD Barcelona systems. + MVAPICH2-1.2 (11/06/2008) * Bugs fixed since MVAPICH2-1.2-rc2 Modified: mvapich2/branches/1.2/maint/Version =================================================================== --- mvapich2/branches/1.2/maint/Version 2008-11-11 20:39:44 UTC (rev 3131) +++ mvapich2/branches/1.2/maint/Version 2008-11-11 22:23:57 UTC (rev 3132) @@ -1 +1 @@ -1.2.0 +1.2.0p1 From sridharj at mvapich.cse.ohio-state.edu Tue Nov 11 18:41:12 2008 From: sridharj at mvapich.cse.ohio-state.edu (sridharj@mvapich.cse.ohio-state.edu) Date: Tue Nov 11 18:41:25 2008 Subject: [mvapich-commit] r3133 - mvapich/trunk/mpid/ch_gen2 Message-ID: <200811112341.mABNfCI2031567@mvapich.cse.ohio-state.edu> Author: sridharj Date: 2008-11-11 18:41:10 -0500 (Tue, 11 Nov 2008) New Revision: 3133 Modified: mvapich/trunk/mpid/ch_gen2/cm.c mvapich/trunk/mpid/ch_gen2/cm.h mvapich/trunk/mpid/ch_gen2/cm_user.c mvapich/trunk/mpid/ch_gen2/mpid_send.c mvapich/trunk/mpid/ch_gen2/viacheck.c mvapich/trunk/mpid/ch_gen2/viapriv.h Log: Bug in replacing indirect (XRC) connection when a real QP is available. Modified: mvapich/trunk/mpid/ch_gen2/cm.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/cm.c 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/cm.c 2008-11-11 23:41:10 UTC (rev 3133) @@ -618,14 +618,14 @@ return 0; } - int cm_accept(cm_msg * msg) { cm_msg msg_send; int flag; /*Prepare QP */ CM_DBG("cm_accpet Enter"); - viadev.qp_hndl[msg->client_rank] = cm_create_rc_qp(msg->client_rank, &flag, 1); + viadev.qp_hndl[msg->client_rank] = cm_create_rc_qp (msg->client_rank, + &flag, 1); #ifdef XRC add_qp_xrc_hash (msg->client_rank, viadev.qp_hndl[msg->client_rank]); #endif @@ -661,7 +661,12 @@ { viadev_conn_queue_t *iter, *tmp; viadev_connection_t *c = &(viadev.connections[owner_rank]); + + /* We now have a direct QP to owner_rank */ + c->xrc_flags |= XRC_FLAG_DIRECT_CONN; + c->xrc_qp_dst = owner_rank; + /* Enable any other connection that may be waiting to use this QP */ iter = c->xrc_conn_queue; while (iter) @@ -692,7 +697,9 @@ cm_state_cli[msg->server_rank] = CM_CONN_STATE_C_IDLE; cm_conn_state[msg->server_rank] = MPICM_IB_RC_PT2PT; #ifdef XRC - cm_activate_pending_xrc_conn (msg->server_rank); + if (viadev_use_xrc) { + cm_activate_pending_xrc_conn (msg->server_rank); + } #endif /* Mark new connection*/ @@ -716,9 +723,8 @@ viadev_connection_t *c = &viadev.connections[msg->client_rank]; if (viadev.qp_hndl[msg->client_rank] != NULL && c->xrc_flags & XRC_FLAG_INDIRECT_CONN) { - c->xrc_flags ^= XRC_FLAG_INDIRECT_CONN; + c->xrc_flags |= XRC_FLAG_NEW_QP; cm_accept (msg); - c->xrc_qp_dst = msg->client_rank; } } #endif @@ -1228,7 +1234,7 @@ return 0; } -int MPICM_Connect_req(int peer_rank, int force) +int MPICM_Connect_req(int peer_rank) { cm_msg req; int flag; @@ -1259,7 +1265,7 @@ CM_DBG("Sending Req to rank %d", peer_rank); cm_state_cli[peer_rank] = CM_CONN_STATE_C_REQUESTING; flag = 0; - viadev.qp_hndl[peer_rank] = cm_create_rc_qp(peer_rank, &flag, force); + viadev.qp_hndl[peer_rank] = cm_create_rc_qp(peer_rank, &flag, 0); if (flag == XRC_QP_NEW) { #ifdef XRC @@ -1302,6 +1308,7 @@ int MPICM_Server_connection_establish(int peer_rank) { cm_pending *pending; + viadev_connection_t *c; MPICM_Lock(); if (cm_state_srv[peer_rank] != CM_CONN_STATE_S_REQUESTED) { /*not waiting for comfirm */ @@ -1321,7 +1328,17 @@ cm_state_srv[peer_rank] = CM_CONN_STATE_S_IDLE; cm_conn_state[peer_rank] = MPICM_IB_RC_PT2PT; #ifdef XRC - cm_activate_pending_xrc_conn (peer_rank); + if (viadev_use_xrc) { + c = &(viadev.connections[peer_rank]); + if (c->xrc_flags & XRC_FLAG_NEW_QP) { + c->xrc_flags &= ~XRC_FLAG_INDIRECT_CONN; + c->xrc_flags ^= XRC_FLAG_NEW_QP; + + /* Replace indirect connection with direct one */ + c->vi = viadev.qp_hndl[peer_rank]; + } + cm_activate_pending_xrc_conn (peer_rank); + } #endif MPICM_Unlock(); Modified: mvapich/trunk/mpid/ch_gen2/cm.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/cm.h 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/cm.h 2008-11-11 23:41:10 UTC (rev 3133) @@ -73,7 +73,7 @@ MPICM_Connect_req Request to establish a type of connection. */ -int MPICM_Connect_req(int peer_rank, int force); +int MPICM_Connect_req(int peer_rank); /* MPICM_Finalize_UD Modified: mvapich/trunk/mpid/ch_gen2/cm_user.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/cm_user.c 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/cm_user.c 2008-11-11 23:41:10 UTC (rev 3133) @@ -75,7 +75,10 @@ #endif #endif - c->vi = viadev.qp_hndl[i]; + /* If I already have (an indirect) connection, continue to use it until + * the new QP is active */ + if (NULL == c->vi) + c->vi = viadev.qp_hndl[i]; if(!viadev_use_srq) { /* prepost receives */ Modified: mvapich/trunk/mpid/ch_gen2/mpid_send.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/mpid_send.c 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/mpid_send.c 2008-11-11 23:41:10 UTC (rev 3133) @@ -108,7 +108,7 @@ /*First pending request */ viadev.pending_req_head[dest_grank] = viadev.pending_req_tail[dest_grank] = req; - MPICM_Connect_req(dest_grank, 0); + MPICM_Connect_req(dest_grank); } else { @@ -308,7 +308,7 @@ /*First pending request */ viadev.pending_req_head[dest_grank] = viadev.pending_req_tail[dest_grank] = req; - MPICM_Connect_req(dest_grank, 0); + MPICM_Connect_req(dest_grank); } else { Modified: mvapich/trunk/mpid/ch_gen2/viacheck.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viacheck.c 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/viacheck.c 2008-11-11 23:41:10 UTC (rev 3133) @@ -1209,11 +1209,10 @@ if ((viadev_rdma_eager_threshold < c->eager_start_cnt) && !viadev.is_finalized) { #ifdef XRC - if (viadev_use_xrc && (c->vi == NULL || - c->xrc_flags & XRC_FLAG_INDIRECT_CONN)) { + if (viadev_use_xrc && c->vi == NULL) { if (!(c->xrc_flags & XRC_FLAG_START_RDMAFP)) { c->xrc_flags |= XRC_FLAG_START_RDMAFP; - MPICM_Connect_req (header->src_rank, 1); + MPICM_Connect_req (header->src_rank); } } else @@ -1254,7 +1253,7 @@ xrc_queue_packet (v, header->src_rank); /* Create / Find RC QP */ - MPICM_Connect_req (header->src_rank, 0); + MPICM_Connect_req (header->src_rank); break; } else Modified: mvapich/trunk/mpid/ch_gen2/viapriv.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/viapriv.h 2008-11-11 22:23:57 UTC (rev 3132) +++ mvapich/trunk/mpid/ch_gen2/viapriv.h 2008-11-11 23:41:10 UTC (rev 3133) @@ -301,6 +301,7 @@ #define XRC_FLAG_START_RDMAFP 0x00000001 #define XRC_FLAG_DIRECT_CONN 0x00000002 #define XRC_FLAG_INDIRECT_CONN 0x00000004 +#define XRC_FLAG_NEW_QP 0x00000008 #endif From koop at mvapich.cse.ohio-state.edu Wed Nov 12 09:01:15 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Wed Nov 12 09:01:28 2008 Subject: [mvapich-commit] r3134 - mvapich/trunk/mpid/ch_hybrid Message-ID: <200811121401.mACE1FpJ013479@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-12 09:01:14 -0500 (Wed, 12 Nov 2008) New Revision: 3134 Modified: mvapich/trunk/mpid/ch_hybrid/mv_connect.c Log: * Correct issue when activating XRC connections that are shared Modified: mvapich/trunk/mpid/ch_hybrid/mv_connect.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/mv_connect.c 2008-11-11 23:41:10 UTC (rev 3133) +++ mvapich/trunk/mpid/ch_hybrid/mv_connect.c 2008-11-12 14:01:14 UTC (rev 3134) @@ -97,10 +97,11 @@ while(NULL != curr_conn) { if(MV_XRC_CONN_SINGLE_SRQN_RECEIVED == curr_conn->xrc_channel.status) { - c->xrc_channel.status = MV_XRC_CONN_SINGLE_ESTABLISHED; - c->xrc_enabled = 1; - if(c->rcfp_send_enabled_xrc_pending) { - MV_Recv_RC_FP_Addr(c, c->pending_rcfp); + + curr_conn->xrc_channel.status = MV_XRC_CONN_SINGLE_ESTABLISHED; + curr_conn->xrc_enabled = 1; + if(curr_conn->rcfp_send_enabled_xrc_pending) { + MV_Recv_RC_FP_Addr(curr_conn, curr_conn->pending_rcfp); } } curr_conn = curr_conn->xrc_channel_ptr; From sridharj at mvapich.cse.ohio-state.edu Fri Nov 14 13:54:34 2008 From: sridharj at mvapich.cse.ohio-state.edu (sridharj@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 13:54:48 2008 Subject: [mvapich-commit] r3136 - in mvapich/trunk/mpid: ch_gen2/process ch_hybrid/process ch_psm/process ch_smp/process Message-ID: <200811141854.mAEIsYFW012323@mvapich.cse.ohio-state.edu> Author: sridharj Date: 2008-11-14 13:54:33 -0500 (Fri, 14 Nov 2008) New Revision: 3136 Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c mvapich/trunk/mpid/ch_hybrid/process/mpirun_rsh.c mvapich/trunk/mpid/ch_psm/process/mpirun_rsh.c mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c Log: Minor: Fix totalview identifier string. Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2008-11-14 00:53:36 UTC (rev 3135) +++ mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2008-11-14 18:54:33 UTC (rev 3136) @@ -176,7 +176,7 @@ int MPIR_proctable_size = 0; int MPIR_i_am_starter = 1; int MPIR_debug_state = 0; -char *MPIR_dll_name = "MVAPICH2"; +char *MPIR_dll_name = "MVAPICH"; /* Totalview intercepts MPIR_Breakpoint */ int MPIR_Breakpoint (void) { Modified: mvapich/trunk/mpid/ch_hybrid/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/process/mpirun_rsh.c 2008-11-14 00:53:36 UTC (rev 3135) +++ mvapich/trunk/mpid/ch_hybrid/process/mpirun_rsh.c 2008-11-14 18:54:33 UTC (rev 3136) @@ -176,7 +176,7 @@ int MPIR_proctable_size = 0; int MPIR_i_am_starter = 1; int MPIR_debug_state = 0; -char *MPIR_dll_name = "MVAPICH2"; +char *MPIR_dll_name = "MVAPICH"; /* Totalview intercepts MPIR_Breakpoint */ int MPIR_Breakpoint (void) { Modified: mvapich/trunk/mpid/ch_psm/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_psm/process/mpirun_rsh.c 2008-11-14 00:53:36 UTC (rev 3135) +++ mvapich/trunk/mpid/ch_psm/process/mpirun_rsh.c 2008-11-14 18:54:33 UTC (rev 3136) @@ -176,7 +176,7 @@ int MPIR_proctable_size = 0; int MPIR_i_am_starter = 1; int MPIR_debug_state = 0; -char *MPIR_dll_name = "MVAPICH2"; +char *MPIR_dll_name = "MVAPICH"; /* Totalview intercepts MPIR_Breakpoint */ int MPIR_Breakpoint (void) { Modified: mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c 2008-11-14 00:53:36 UTC (rev 3135) +++ mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c 2008-11-14 18:54:33 UTC (rev 3136) @@ -176,7 +176,7 @@ int MPIR_proctable_size = 0; int MPIR_i_am_starter = 1; int MPIR_debug_state = 0; -char *MPIR_dll_name = "MVAPICH2"; +char *MPIR_dll_name = "MVAPICH"; /* Totalview intercepts MPIR_Breakpoint */ int MPIR_Breakpoint (void) { From perkinjo at mvapich.cse.ohio-state.edu Fri Nov 14 14:54:33 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 14:54:46 2008 Subject: [mvapich-commit] r3137 - mvapich/trunk Message-ID: <200811141954.mAEJsXrV012439@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-14 14:54:32 -0500 (Fri, 14 Nov 2008) New Revision: 3137 Modified: mvapich/trunk/CHANGELOG Log: Update CHANGELOG for mvapich-1.1 release. Modified: mvapich/trunk/CHANGELOG =================================================================== --- mvapich/trunk/CHANGELOG 2008-11-14 18:54:33 UTC (rev 3136) +++ mvapich/trunk/CHANGELOG 2008-11-14 19:54:32 UTC (rev 3137) @@ -4,6 +4,39 @@ This file briefly describes the latest changes to MVAPICH software package. The logs are arranged in the "most recent first" order. +11/12/2008 +* Correct issue when activating XRC connections that are shared. + +11/04/2008 +* Increase the number of allowed nodes for shared memory broadcast to 4K. + +10/30/2008 +* Increasing the threshold at which we switch from Recursive Doubling to Ring + algorithm for Allgather collective. + +10/29/2008 +* Ignore the last bit of the pkey +* Remove the pkey_ix option since the index + can be different on different machines +* Processes can be separated by ":" for cpu mapping to be consistent with + mvapich2. Processes can also be separated by "," for backward compatibility. + +10/25/2008 +* Diskless shared memory files. + +10/20/2008 +* Enhanced TotalView support. + +10/09/2008 +* Implement MPI_Query_thread and MPI_Is_thread_main. + +10/08/2008 +* Fix a bug in allreduce. Do not call the user-defined function with 0 length. + +10/05/2008 +* Move XRC finalization until after SRQs are destroyed. + Thanks to Pasha at Mellanox (OpenFabrics: #1244). + 09/27/2008 * Redirect stdin of non-root mpi procs to /dev/null. From perkinjo at mvapich.cse.ohio-state.edu Fri Nov 14 16:21:53 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 16:22:05 2008 Subject: [mvapich-commit] r3138 - mvapich/branches Message-ID: <200811142121.mAELLrwh012715@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-14 16:21:52 -0500 (Fri, 14 Nov 2008) New Revision: 3138 Added: mvapich/branches/1.1/ Log: Create mvapich 1.1 branch Copied: mvapich/branches/1.1 (from rev 3137, mvapich/trunk) From perkinjo at mvapich.cse.ohio-state.edu Fri Nov 14 16:22:25 2008 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 16:22:37 2008 Subject: [mvapich-commit] r3139 - mvapich/tags Message-ID: <200811142122.mAELMPIl012733@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2008-11-14 16:22:25 -0500 (Fri, 14 Nov 2008) New Revision: 3139 Added: mvapich/tags/1.1/ Log: Create tag for mvapich-1.1 release Copied: mvapich/tags/1.1 (from rev 3138, mvapich/trunk) From koop at mvapich.cse.ohio-state.edu Fri Nov 14 19:50:48 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 19:50:59 2008 Subject: [mvapich-commit] r3140 - mvapich/tags Message-ID: <200811150050.mAF0omkh013482@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-14 19:50:47 -0500 (Fri, 14 Nov 2008) New Revision: 3140 Added: mvapich/tags/release-1.1/ Log: Tagging the release Copied: mvapich/tags/release-1.1 (from rev 3139, mvapich/trunk) From koop at mvapich.cse.ohio-state.edu Fri Nov 14 19:51:24 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Fri Nov 14 19:51:36 2008 Subject: [mvapich-commit] r3141 - mvapich/tags Message-ID: <200811150051.mAF0pOQ0013500@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-14 19:51:24 -0500 (Fri, 14 Nov 2008) New Revision: 3141 Removed: mvapich/tags/1.1/ Log: * remove old tag From koop at mvapich.cse.ohio-state.edu Sun Nov 23 19:53:53 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Sun Nov 23 19:54:07 2008 Subject: [mvapich-commit] r3142 - mvapich/trunk/mpid/ch_hybrid Message-ID: <200811240053.mAO0rrtN021634@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-23 19:53:52 -0500 (Sun, 23 Nov 2008) New Revision: 3142 Modified: mvapich/trunk/mpid/ch_hybrid/mv_init.c Log: * Move SMP/affinity initalization before buffers are allocated - greatly increases performance for hybrid device on NUMA systems Modified: mvapich/trunk/mpid/ch_hybrid/mv_init.c =================================================================== --- mvapich/trunk/mpid/ch_hybrid/mv_init.c 2008-11-15 00:51:24 UTC (rev 3141) +++ mvapich/trunk/mpid/ch_hybrid/mv_init.c 2008-11-24 00:53:52 UTC (rev 3142) @@ -312,7 +312,6 @@ pid_t pid; int hca_lid; int ud_qpn; - int host_id; } mv_exchange_info; mv_exchange_info my_info, *all_info; @@ -321,20 +320,16 @@ my_info.pid = getpid(); my_info.hca_lid = mvdev.default_hca->default_port_attr->lid; my_info.ud_qpn = mvdev.ud_qp[0].qp->qp_num; - my_info.host_id = AUX_Get_HostID(mvdev.my_name); all_info = (mv_exchange_info *) malloc(sizeof(mv_exchange_info) * mvdev.np); pmgr_allgather(&my_info, sizeof(mv_exchange_info), all_info); - hostnames_od = (int *) malloc(mvdev.np * sizeof(int)); - for(i = 0; i < mvdev.np; i++) { mvdev.lids[i] = all_info[i].hca_lid; mvdev.qpns[i] = all_info[i].ud_qpn; mvdev.pids[i] = all_info[i].pid; - hostnames_od[i] = all_info[i].host_id; } #ifdef XRC @@ -346,13 +341,7 @@ } #endif -#ifdef _SMP_ - if(!disable_shared_mem) { - odu_init_SMP(); - } -#endif - free(hostnames_od); free(all_info); } @@ -379,6 +368,22 @@ AUX_Get_Hostname(); MV_Init_Params(mvdev.np, mvdev.me); + mvdev.connections = (mvdev_connection_t *) + malloc(sizeof(mvdev_connection_t) * mvdev.np); + + +#ifdef _SMP_ + int host_id = AUX_Get_HostID(mvdev.my_name); + hostnames_od = (int *) malloc(mvdev.np * sizeof(int)); + pmgr_allgather(&host_id, sizeof(int), hostnames_od); + + if(!disable_shared_mem) { + odu_init_SMP(); + } + + free(hostnames_od); +#endif + /* need to figure out if reliability is being used */ mvdev_header_size = MV_HEADER_NORM_OFFSET_SIZE; @@ -400,9 +405,6 @@ mvdev.lids = (uint32_t *) malloc(sizeof(uint32_t) * mvdev.np); mvdev.qpns = (uint32_t *) malloc(sizeof(uint32_t) * mvdev.np); - mvdev.connections = (mvdev_connection_t *) - malloc(sizeof(mvdev_connection_t) * mvdev.np); - mvdev.connection_ack_required = (uint8_t *) malloc(sizeof(uint8_t) * mvdev.np); From koop at mvapich.cse.ohio-state.edu Sun Nov 23 23:12:13 2008 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Sun Nov 23 23:38:08 2008 Subject: [mvapich-commit] r3143 - in mvapich/trunk: . src/coll src/context src/env Message-ID: <200811240412.mAO4CDlB022137@mvapich.cse.ohio-state.edu> Author: koop Date: 2008-11-23 23:12:11 -0500 (Sun, 23 Nov 2008) New Revision: 3143 Removed: mvapich/trunk/src/coll/intra_rdma_allgather.c mvapich/trunk/src/coll/intra_rdma_alltoall.c mvapich/trunk/src/context/comm_rdma_init.c Modified: mvapich/trunk/make.mvapich.hybrid mvapich/trunk/src/coll/Makefile.in mvapich/trunk/src/coll/intra_fns_new.c mvapich/trunk/src/context/Makefile.in mvapich/trunk/src/context/comm_free.c mvapich/trunk/src/context/comm_util.c mvapich/trunk/src/env/initutil.c Log: * Remove RDMA collectives Modified: mvapich/trunk/make.mvapich.hybrid =================================================================== --- mvapich/trunk/make.mvapich.hybrid 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/make.mvapich.hybrid 2008-11-24 04:12:11 UTC (rev 3143) @@ -75,7 +75,7 @@ HAVE_MPD_RING=${HAVE_MPD_RING:-} # Set this to override automatic optimization setting (-03). -OPT_FLAG=${OPT_FLAG:--O3 -g} +OPT_FLAG=${OPT_FLAG:--O3} #OPT_FLAG=-g export LIBS=${LIBS:--L${IBHOME_LIB} -Wl,-rpath=${IBHOME_LIB} -libverbs -lpthread -lm} Modified: mvapich/trunk/src/coll/Makefile.in =================================================================== --- mvapich/trunk/src/coll/Makefile.in 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/coll/Makefile.in 2008-11-24 04:12:11 UTC (rev 3143) @@ -33,7 +33,7 @@ allgather.c allgatherv.c alltoall.c alltoallv.c reduce.c \ allreduce.c red_scat.c scan.c global_ops.c \ opcreate.c opfree.c oputil.c inter_fns.c ${MPI_INTRA}.c \ - intra_scan.c intra_rdma_alltoall.c intra_rdma_allgather.c + intra_scan.c MPI_TSOURCES = global_ops.c @@ -41,7 +41,7 @@ allgather.o allgatherv.o alltoall.o alltoallv.o reduce.o \ allreduce.o red_scat.o scan.o global_ops.o \ opcreate.o opfree.o oputil.o inter_fns.o ${MPI_INTRA}.o \ - intra_scan.o intra_rdma_alltoall.o intra_rdma_allgather.o + intra_scan.o default: ${libbuild_dir}/lib@MPILIBNAME@.a Modified: mvapich/trunk/src/coll/intra_fns_new.c =================================================================== --- mvapich/trunk/src/coll/intra_fns_new.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/coll/intra_fns_new.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -61,7 +61,6 @@ #include "binding.h" #endif -extern int enable_rdma_collectives; #ifdef _SMP_ extern int enable_shmem_collectives; int disable_shmem_barrier=0; @@ -91,13 +90,6 @@ extern int HEADER; #endif -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) -extern int intra_RDMA_barrier(struct MPIR_COMMUNICATOR *); -extern int intra_rdma_Alltoall(void *, int, struct MPIR_DATATYPE *, - void *, int, struct MPIR_DATATYPE *, - struct MPIR_COMMUNICATOR *, int); -#endif - #ifdef _SMP_ extern void MPID_SHMEM_COLL_Barrier_gather(int, int, int); extern void MPID_SHMEM_COLL_Barrier_bcast(int, int, int); @@ -237,10 +229,6 @@ MPI_Op op, struct MPIR_COMMUNICATOR *comm ); #endif -int intra_rdma_Allgather(void *, int , struct MPIR_DATATYPE *, - void *, int recvcnt, struct MPIR_DATATYPE *, - struct MPIR_COMMUNICATOR *, int); - /* I don't really want to to this this way, but for now... */ static struct _MPIR_COLLOPS intra_collops = { #ifdef MPID_Barrier @@ -356,10 +344,6 @@ int d, dst, src; MPI_Status status; - int rdma_enabled; - - rdma_enabled = 0; - /* Intialize communicator size */ (void) MPIR_Comm_size ( comm, &size ); @@ -375,32 +359,6 @@ /* Initialize collective communicator */ comm = comm->comm_coll; - if (enable_rdma_collectives){ - /* Switching over to rdma based barrier if the corresponding macro is - * defined */ -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) - if(comm->is_barrier_enabled) { - rdma_enabled = 1; - } else { - rdma_enabled = 0; - } -#endif /* VIADEV_RPUT_SUPPORT */ - -#ifdef CH_GEN2_MRAIL - rdma_enabled = 0; -#endif - if (rdma_enabled) { - -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) - if (!viadev_use_on_demand) { - if ( comm->np >2 ){ - (void) intra_RDMA_barrier( comm ); - return (MPI_SUCCESS); - } - } -#endif /* VIADEV_RPUT_SUPPORT */ - } - } (void) MPIR_Comm_rank ( comm, &rank ); (void) MPIR_Comm_N2_prev ( comm, &N2_prev ); surfeit = size - N2_prev; @@ -2880,7 +2838,7 @@ send_offset, recv_offset, last_recv_cnt, nprocs_completed, k, offset, tmp_mask, tree_root, tmp_buf_size, position, nbytes; - int rdma_allgather_enabled, is_contig, sendtype_size; + int is_contig, sendtype_size; void *send_mybuf, *recv_mybuf; int mylen; struct MPIR_COMMUNICATOR *new_comm_ptr; @@ -2935,48 +2893,6 @@ /* Lock for collective operation */ MPID_THREAD_LOCK(comm->ADIctx,comm); -#if ((defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) \ - && (!defined(MAC_OSX)) && (!defined(CH_GEN2_MRAIL)) && (!defined(USE_MRAIL))) - if(comm->is_allgather_enabled && !viadev_use_on_demand) { - rdma_allgather_enabled = 1; - } else { - rdma_allgather_enabled = 0; - } - - - if((enable_rdma_collectives) && (rdma_allgather_enabled) && (size > 2) && size_is_pof2 && - (sendtype->dte_type <= 12) && !viadev_use_on_demand && - ((unsigned long) nbytes*size < viadev_pt2pt_failover)) { - if (is_contig){ - mpi_errno = intra_rdma_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, nbytes); - } - else{ - MPID_PackMessage(sendbuf, sendcount, sendtype, - comm, 0, msgrep, msgact, (void **) &send_mybuf, &mylen, &mpi_errno); - assert(mylen == nbytes); - if (mpi_errno) return mpi_errno; - MPIR_Datatype_iscontig(recvtype->self, &is_contig); - if (is_contig){ - mpi_errno = intra_rdma_Allgather(send_mybuf, sendcount, - sendtype, recvbuf, recvcount, recvtype, comm, nbytes); - } - else{ - MPID_UnpackMessageSetup(recvcount, recvtype, comm, 0, msgrep, - (void **) &recv_mybuf, &mylen, &mpi_errno); - if (mpi_errno) return mpi_errno; - mpi_errno = intra_rdma_Allgather(send_mybuf, sendcount, - sendtype, recv_mybuf, recvcount, recvtype, comm, nbytes); - MPID_Unpack(recv_mybuf, nbytes*size, MPID_MSGREP_UNKNOWN, - &in_position, recvbuf, recvcount*size, recvtype, - &out_position, (struct MPIR_COMMUNICATOR *) NULL, - 0, &mpi_errno); - - if (mpi_errno) return mpi_errno; - } - } - } - else -#endif if (recvcount*size*type_size < allgather_large_msg_threshold) { /* * Trac #208, #144 @@ -3979,7 +3895,7 @@ void *tmp_buf; MPI_Status *starray; MPI_Request *reqarray; - int rdma_enabled, is_contig, use_rdma_flag = 0; + int is_contig; #ifdef OLD MPI_Aint sendbuf_extent; @@ -3989,8 +3905,6 @@ if (sendcount == 0) return MPI_SUCCESS; - rdma_enabled = 0; - /* Get size and switch to collective communicator */ MPIR_Comm_size ( comm, &size ); MPIR_Comm_rank ( comm, &rank ); @@ -4008,39 +3922,8 @@ /* Lock for collective operation */ MPID_THREAD_LOCK(comm->ADIctx, comm); -#if ((defined(VIADEV_RPUT_SUPPORT) || (defined(VIADEV_RPUT_SUPPORT))) \ - && (!defined(MAC_OSX)) && (!defined(SOLARIS))) - if(comm->is_alltoall_enabled) { - rdma_enabled = 1; - } else { - rdma_enabled = 0; - } - -#endif /* VIADEV_RPUT_SUPPORT */ - -#ifdef CH_GEN2_MRAIL - rdma_enabled = 0; -#endif -#if ((defined(VIADEV_RPUT_SUPPORT)||(defined(VIADEV_RGET_SUPPORT)))) - if((enable_rdma_collectives)&&(rdma_enabled) && !viadev_use_on_demand && - (nbytes >= viadev_rendezvous_threshold) && -#if (!defined(CH_GEN2_MRAIL) && !defined(USE_MRAIL) ) - ((unsigned long) nbytes*size < viadev_pt2pt_failover) && -#endif - (size > 2) && - is_contig && - (sendtype->dte_type <= 12) ) { - /* If the user has enabled RDMA alltoall, then - * use it. */ - use_rdma_flag = 1; - mpi_errno = intra_rdma_Alltoall(sendbuf, sendcount, - sendtype, recvbuf, recvcnt, recvtype, comm, - nbytes); - } -#endif /* VIADEV_RPUT_SUPPORT */ - if ((nbytes <= mpir_alltoall_short_msg) && (size >= 8) - && !use_rdma_flag) { + if ((nbytes <= mpir_alltoall_short_msg) && (size >= 8)) { /* use the indexing algorithm by Jehoshua Bruck et al, * IEEE TPDS, Nov. 97 */ @@ -4303,7 +4186,7 @@ } - else if (nbytes <= mpir_alltoall_medium_msg && !(use_rdma_flag)) { + else if (nbytes <= mpir_alltoall_medium_msg) { /* 1st, get some storage from the heap to hold handles, etc. */ MPIR_ALLOC(starray,(MPI_Status *)MALLOC(2*size*sizeof(MPI_Status)), @@ -4352,7 +4235,7 @@ FREE(reqarray); } - else if (!(use_rdma_flag)) { + else { /* Long message. Use pairwise exchange. If comm_size is a power-of-two, use exclusive-or to create pairs. Else send to rank+i, receive from rank-i. */ Deleted: mvapich/trunk/src/coll/intra_rdma_allgather.c =================================================================== --- mvapich/trunk/src/coll/intra_rdma_allgather.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/coll/intra_rdma_allgather.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -1,369 +0,0 @@ -/* Copyright (c) 2002-2008, The Ohio State University. All rights - * reserved. - * - * This file is part of the MVAPICH software package, developed by the - * team members of The Ohio State University's Network-Based Computing - * Laboratory (NBCL) and headed by Professor Dhabaleswar K. (DK) Panda. - * - * For detailed copyright and licensing information, please refer to the - * copright file COPYRIGHT_MVAPICH in the top level MPICH directory. - * - */ - -#if ( (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) \ - && (!defined(USE_MRAIL)) && (!defined(CH_GEN2_MRAIL))) - -#include "mpid.h" -#include "mpiimpl.h" -#include "mpimem.h" - -#if defined(CH_GEN2) -#include "../../mpid/ch_gen2/collutils.h" -#else -#include "../../mpid/vapi/collutils.h" -#endif - -static inline void allgather_all_send_complete(int, int); - -int intra_rdma_Allgather(void *sendbuf, - int sendcount, - struct MPIR_DATATYPE *sendtype, - void *recvbuf, - int recvcount, - struct MPIR_DATATYPE *recvtype, - struct MPIR_COMMUNICATOR *comm, int one_size) -{ - int dest, dimensions, mask, subcube_mask, rank, size, i, j; - int alternate, next_alternate, round; - int size_one_send_buf, size_one_recv_buf, virtual_colbuf_offset; - int *peer_colbuf_offset = NULL; - int timer=0,max_spin_time=10000; - struct Collbuf *collbuf; - char *local_addr, *dest_addr, *recv_addr; - MPI_Aint sendtype_extent, recvtype_extent; - void *poll_addr; - char *local_buf = NULL; - unsigned long allgather_msg_size, len; - static unsigned long prev_allgather_msg_size = 0; - static unsigned long clear_allgather_msg_size = 0; - struct Allgather *allgather; - struct Allgather_Peer *peers; - struct L_Collbuf *l_coll; - struct R_Collbuf *r_coll; - struct MPIR_COMMUNICATOR *actual_comm; - dreg_entry *d_recv = NULL; - - /* Get size and switch to collective communicator */ - MPIR_Comm_size(comm, &size); - MPIR_Comm_rank(comm, &rank); - - actual_comm = comm; - - comm = comm->comm_coll; - - if (comm->collbuf == NULL) { - /* No proper initialization */ - MPID_Abort(comm, -1, "Illegal collective", NULL); - } - - /* Get extent of send and recv types */ - MPI_Type_extent(sendtype->self, &sendtype_extent); - MPI_Type_extent(recvtype->self, &recvtype_extent); - - size_one_send_buf = sendcount*sendtype_extent; - size_one_recv_buf = recvcount*recvtype_extent; - - /* 0 byte Allgather ? */ - if (size_one_send_buf <= 0) { - return MPI_SUCCESS; - } - - collbuf = comm->collbuf; - /* Adjusting the offset for the allgather buffer*/ - - l_coll = (collbuf->l_coll) ; - r_coll = collbuf->r_coll; - allgather = collbuf->allgather; - dimensions = allgather->dimensions; - peers = allgather->peers; - peer_colbuf_offset = (int*) calloc(sizeof(int), dimensions); - - if(peer_colbuf_offset == NULL) { - fprintf(stderr, "[%d] Failed to allocate peer_colbuf_offset file: %s, line: %d\n", - rank, __FILE__, __LINE__); - - return MPI_ERR_UNKNOWN; - } - - - /* To support back-to-back allgather collective operations, - * we use three back-to-back buffers; no need to wait for sends to - * complete except to ensure progress */ - alternate = allgather->alternate; - next_alternate = (alternate+1)%3; - allgather->alternate = next_alternate; - - allgather_msg_size = size_one_send_buf + ALLGATHER_HEADER_SIZE; - - /* 1. Clear next b2b buffer as required */ - if (clear_allgather_msg_size > ALLGATHER_THRESHOLD) { - /* just clear the flags used for completion RDMA writes and - * the space used for address as we had used only zero-copy */ - memset(l_coll->buf + ALLGATHER_OFFSET + next_alternate*COLL_SINGLE_BUF_SIZE - + ALLGATHER_BUF_SIZE, 0, - COLL_FLAG_SPACE + COLL_ADDR_RECV_SPACE); - }else{ - memset(l_coll->buf + ALLGATHER_OFFSET + next_alternate*COLL_SINGLE_BUF_SIZE, - 0, MIN((clear_allgather_msg_size+ALLGATHER_HEADER_SIZE)*size, - COLL_SINGLE_BUF_SIZE)); - - if (clear_allgather_msg_size*size/2 > ALLGATHER_THRESHOLD) { - /* we used zero-copy also */ - memset(l_coll->buf + ALLGATHER_OFFSET + next_alternate*COLL_SINGLE_BUF_SIZE - + ALLGATHER_BUF_SIZE, 0, - COLL_FLAG_SPACE + COLL_ADDR_RECV_SPACE); - } - } - local_buf = l_coll->buf + ALLGATHER_OFFSET + alternate*COLL_SINGLE_BUF_SIZE; - - /* 2. Register the recv buffer and copy the recv buffer address - * to the collective buffer to be sent out if required */ - if (size_one_recv_buf*size/2 > ALLGATHER_THRESHOLD) { - - /* Register the user buffer */ - d_recv = dreg_register(recvbuf, size_one_recv_buf*comm->np, - DREG_ACL_WRITE); - - if (NULL == d_recv) { - fprintf(stderr, "[%d] Registration failed, file: %s, line: %d\n", - rank, __FILE__, __LINE__); - return MPI_ERR_UNKNOWN; - } - - /* copy your user buffer address to collective buffer */ -#if defined(CH_GEN2) - copy_recv_addr_allgather(recvbuf, collbuf, d_recv->memhandle); -#else - copy_recv_addr_allgather(recvbuf, collbuf, &(d_recv->memhandle)); -#endif - } - - /* 3. Send address of user buffer if required and that too to only - * necessary peers */ - virtual_colbuf_offset = 0; - subcube_mask = (1 << dimensions) - 1; - for(mask=0x1,round=0; mask ALLGATHER_THRESHOLD) { - if (0 == virtual_colbuf_offset) { - virtual_colbuf_offset = (rank&subcube_mask)*allgather_msg_size; - for (j=0x1, i=0; ilrank_to_grank[dest], - allgather->buf_hndl_addr, - (allgather->buf_hndl_addr_len + - allgather->rdma_recv_addr_len + sizeof(char)), - l_coll->mr, - peers[round].dest_addr[alternate], - (r_coll[dest].rkey)); -#else - coll_rdma_write(actual_comm->lrank_to_grank[dest], - allgather->buf_hndl_addr, - allgather->buf_hndl_addr_len + - allgather->rdma_recv_addr_len + - sizeof(char), - &(l_coll->buf_hndl), - peers[round].dest_addr[alternate], - &(r_coll[dest].buf_hndl)); -#endif - } - subcube_mask <<= 1; - } - - local_buf = local_buf - virtual_colbuf_offset; - - /* 4. Copy over my message to my collective buffer and recv buffer */ - if (size_one_recv_buf <= ALLGATHER_THRESHOLD) { - copy_to_rdma_buf(local_buf + rank*allgather_msg_size, sendbuf, - size_one_send_buf, rank, rank); - } - memcpy(recvbuf + rank*size_one_recv_buf, sendbuf, size_one_recv_buf); - - - /* 5. Do allgather using recursive doubling */ - subcube_mask = (1 << dimensions) - 1; - - for(mask=0x1, round=0; masklrank_to_grank[dest], - local_addr, len, - (l_coll->mr), dest_addr, - (r_coll[dest].rkey)); -#else - - coll_rdma_write(actual_comm->lrank_to_grank[dest], - local_addr, len, &(l_coll->buf_hndl), - dest_addr, &(r_coll[dest].buf_hndl)); -#endif - - /* overlap progress with largest send. Largest send occurs in - * last iteration: of p/2 messages */ - if (mask == size/2) MPID_DeviceCheck(MPID_NOTBLOCKING); - - /* Recursive Doubling is a combining algo. Data from all - * previous iterations is needed for a given iteration */ - recv_addr = local_buf + (dest & subcube_mask)*allgather_msg_size; - poll_addr = recv_addr + len - sizeof(char); - - timer = 0; - while(*((volatile char *)poll_addr) != ALLGATHER_TAIL_FLAG){ - if (++timer >= max_spin_time){ - timer=0; - MPID_DeviceCheck(MPID_NOTBLOCKING); - } - } - - /* Copy over the data to receive buffer now itself */ - for(i=0; i= max_spin_time){ - timer=0; - MPID_DeviceCheck(MPID_NOTBLOCKING); - } - } - - /* Reset Poll flag */ - *(char *) poll_addr = 0; - - /* Got his address; send him the data */ - local_addr = recvbuf + (rank & subcube_mask)*size_one_recv_buf; - -#if defined(CH_GEN2) - peers[round].mr.rkey = - *((uint32_t*) peers[round].recv_addr[alternate]); - dest_addr = (char *)(*(aint_t *)(peers[round].recv_addr[alternate] - + sizeof(uint32_t))); -#else - peers[round].memhandle.rkey = - *((VAPI_rkey_t *) peers[round].recv_addr[alternate]); - dest_addr = (char *)(*(aint_t *)(peers[round].recv_addr[alternate] - + sizeof(VAPI_rkey_t))); -#endif - dest_addr = dest_addr + (rank & subcube_mask)*size_one_recv_buf; - - /* write the data */ -#if defined(CH_GEN2) - coll_rdma_write(actual_comm->lrank_to_grank[dest], - local_addr, len, (d_recv->memhandle), dest_addr, - (peers[round].mr.rkey)); -#else - - coll_rdma_write(actual_comm->lrank_to_grank[dest], - local_addr, len, &(d_recv->memhandle), - dest_addr, &(peers[round].memhandle)); -#endif - - /* Indicate completion thru b2b RDMA write */ -#if defined(CH_GEN2) - coll_rdma_write(actual_comm->lrank_to_grank[dest], - allgather->rdma_flag, - allgather->rdma_flag_size, - (l_coll->mr), - peers[round].r_done[alternate], - (r_coll[dest].rkey)); -#else - coll_rdma_write(actual_comm->lrank_to_grank[dest], - allgather->rdma_flag, - allgather->rdma_flag_size, - &(l_coll->buf_hndl), - peers[round].r_done[alternate], - &(r_coll[dest].buf_hndl)); -#endif - - if (mask == size/2) MPID_DeviceCheck(MPID_NOTBLOCKING); - - poll_addr = peers[round].done[alternate]; - timer=0; - while(*((volatile char *)poll_addr) != ALLGATHER_RDMA_FLAG){ - if (++timer >= max_spin_time){ - timer=0; - MPID_DeviceCheck(MPID_NOTBLOCKING); - } - } - } - subcube_mask <<= 1; - } - - clear_allgather_msg_size = prev_allgather_msg_size; - prev_allgather_msg_size = size_one_send_buf; - - /* 6. Unregister the user buffer if you had RDMA written from/to it */ - if (size_one_recv_buf*size/2 > ALLGATHER_THRESHOLD) { - allgather_all_send_complete(rank, size); - dreg_unregister(d_recv); - } - - free(peer_colbuf_offset); - - return (MPI_SUCCESS); -} - - -static inline void allgather_all_send_complete(int rank, int size) -{ - unsigned mask; - int not_complete; - viadev_connection_t *c = NULL; - - do{ - not_complete = 0; - for (mask=0x1; masksend_wqes_avail != viadev_sq_size) { - not_complete = 1; - break; - } - } - if (not_complete) MPID_DeviceCheck(MPID_NOTBLOCKING); - }while (not_complete); -} - -#endif /* RDMA_ALLGATHER */ Deleted: mvapich/trunk/src/coll/intra_rdma_alltoall.c =================================================================== --- mvapich/trunk/src/coll/intra_rdma_alltoall.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/coll/intra_rdma_alltoall.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -1,268 +0,0 @@ -/* Copyright (c) 2002-2008, The Ohio State University. All rights - * reserved. - * - * This file is part of the MVAPICH software package, developed by the - * team members of The Ohio State University's Network-Based Computing - * Laboratory (NBCL) and headed by Professor Dhabaleswar K. (DK) Panda. - * - * For detailed copyright and licensing information, please refer to the - * copright file COPYRIGHT_MVAPICH in the top level MPICH directory. - * - */ - -#if ((defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) && (!defined(CH_GEN2_MRAIL))) - -#include "mpid.h" -#include "mpiimpl.h" -#include "mpimem.h" - -#ifdef USE_MRAIL -#include "../../mpid/vapi_multirail/collutils.h" -#elif defined(CH_GEN2) -#include "../../mpid/ch_gen2/collutils.h" -#else -#include "../../mpid/vapi/collutils.h" -#endif - -int intra_rdma_Alltoall(void *sendbuf, - int sendcount, - struct MPIR_DATATYPE *sendtype, - void *recvbuf, - int recvcnt, - struct MPIR_DATATYPE *recvtype, - struct MPIR_COMMUNICATOR *comm, int one_size) -{ - int size, i, rank, size_one_send_buf, size_one_recv_buf; - MPI_Aint sendtype_extent, recvtype_extent; - int sendtype_size; - dreg_entry *d_send = NULL; - dreg_entry *d_recv = NULL; - int dest_rank, alternate, next_alternate; - struct MPIR_COMMUNICATOR *actual_comm; - - struct R_Collbuf *r_coll; - struct L_Collbuf *l_coll; - struct Alltoall *atoa; - struct DE_Remote_Node *node; - - void *poll_addr_ptr; - void *dest_addr; - void *local_addr; - - - /* Get size and switch to collective communicator */ - MPIR_Comm_size(comm, &size); - MPIR_Comm_rank(comm, &rank); - - actual_comm = comm; - comm = comm->comm_coll; - - if (comm->collbuf == NULL) { - /* No proper initialization */ - MPID_Abort(comm, -1, "Illegal collective", NULL); - } - - /* Get extent of send and recv types */ - MPI_Type_extent(sendtype->self, &sendtype_extent); - MPI_Type_extent(recvtype->self, &recvtype_extent); - - MPI_Type_size(sendtype->self, &sendtype_size); - - size_one_send_buf = sendcount * sendtype_size; - size_one_recv_buf = recvcnt * recvtype_extent; - - /* 0 byte Alltoall ? */ - if (size_one_send_buf <= 0) { - return MPI_SUCCESS; - } - - r_coll = comm->collbuf->r_coll; - l_coll = comm->collbuf->l_coll; - atoa = comm->collbuf->atoa; - node = atoa->de->node; - alternate = atoa->de->alternate; - - /* Toggle */ - atoa->de->alternate = (alternate ^ 0x01); - - next_alternate = atoa->de->alternate; - - /* 1. Register the receive buffer */ - d_recv = dreg_register(recvbuf, size_one_recv_buf * comm->np, - DREG_ACL_WRITE); - - if (NULL == d_recv) { - fprintf(stderr, "[%d] Registration failed, file : %s, line : %d\n", - rank, __FILE__, __LINE__); - return MPI_ERR_UNKNOWN; - } - - /* 2 . Copy your recv addr & key into the pre-registered buffer */ -#if defined(CH_GEN2) - copy_recv_addr(recvbuf, comm->collbuf, d_recv->memhandle); -#else - copy_recv_addr(recvbuf, comm->collbuf, &(d_recv->memhandle)); -#endif - - /* 3. Send your recv addr to your peers */ - for (i = 0; i < size; i++) { - dest_rank = (rank + i) % size; - - /* Don't send it to myself ! */ - if (dest_rank == rank) - continue; -#if defined(CH_GEN2) - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - atoa->buf_hndl_addr, - (atoa->buf_hndl_addr_len + - atoa->rdma_recv_addr_len + sizeof(char)), - l_coll->mr, - node[dest_rank].dest_addr[alternate], - r_coll[dest_rank].rkey); -#else - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - atoa->buf_hndl_addr, - (atoa->buf_hndl_addr_len + - atoa->rdma_recv_addr_len + sizeof(char)), - &(l_coll->buf_hndl), - node[dest_rank].dest_addr[alternate], - &(r_coll[dest_rank].buf_hndl)); - -#endif - } - - /* 4. Set the alternate buffers to be zero. */ - for (i = 0; i < size; i++) { - if (i == rank) - continue; - - memset(node[i].recv_addr[next_alternate], 0, - atoa->buf_hndl_addr_len + atoa->rdma_recv_addr_len - + sizeof(char)); - - *((char *) node[i].done[next_alternate]) = 0; - } - - /* 5. Copy over my own buffer from send to recv */ - memcpy((void *) ((char *) recvbuf + (size_one_recv_buf * rank)), - (void *) ((char *) sendbuf + (size_one_send_buf * rank)), - size_one_send_buf); - - /* 6. Register the send buffer */ - d_send = dreg_register(sendbuf, size_one_send_buf * comm->np, - DREG_ACL_WRITE); - - if (NULL == d_send) { - fprintf(stderr, "[%d] Registration failed, file : %s, line : %d\n", - rank, __FILE__, __LINE__); - return MPI_ERR_UNKNOWN; - } - - /* 7. Wait for other peers's addresses */ - for (i = (size - 1); i >= 0; i--) { - dest_rank = (rank + i) % size; - - if (dest_rank == rank) - continue; - -#if defined(CH_GEN2) - poll_addr_ptr = - (void *) ((char *) node[dest_rank].recv_addr[alternate] + - sizeof(uint32_t) + sizeof(aint_t)); -#else - poll_addr_ptr = - (void *) ((char *) node[dest_rank].recv_addr[alternate] + - sizeof(VAPI_rkey_t) + sizeof(aint_t)); -#endif - while (*((volatile char *) poll_addr_ptr) == 0); - - - /* Fill in the data structures */ -#if defined(CH_GEN2) - node[dest_rank].mr.rkey = - *((uint32_t *) node[dest_rank].recv_addr[alternate]); - - dest_addr = - (void *) ((char *) node[dest_rank].recv_addr[alternate] + - sizeof(uint32_t)); -#else - node[dest_rank].memhandle.rkey = - *((VAPI_rkey_t *) node[dest_rank].recv_addr[alternate]); - - dest_addr = - (void *) ((char *) node[dest_rank].recv_addr[alternate] + - sizeof(VAPI_rkey_t)); -#endif - - dest_addr = (void *) (*(aint_t *) dest_addr); - - /* Got this guy's address, should send him - * my data immediately */ - - /* My index in his buffer should be given by - * my rank */ - dest_addr = (void *) ((char *) dest_addr + - rank * recvcnt * recvtype_extent); - - /* The local index should be given by - * his rank */ - local_addr = (void *) ((char *) sendbuf + - dest_rank * sendcount * sendtype_extent); - - /* RDMA write */ -#if defined(CH_GEN2) - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - local_addr, - size_one_send_buf, - d_send->memhandle, dest_addr, - node[dest_rank].mr.rkey); -#else - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - local_addr, - size_one_send_buf, - &(d_send->memhandle), dest_addr, - &(node[dest_rank].memhandle)); -#endif - /* RDMA write to tell him that I - * am done with him */ -#if defined(CH_GEN2) - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - atoa->rdma_flag, - atoa->rdma_flag_size, - l_coll->mr, - node[dest_rank].r_done[alternate], - r_coll[dest_rank].rkey); -#else - coll_rdma_write(actual_comm->lrank_to_grank[dest_rank], - atoa->rdma_flag, - atoa->rdma_flag_size, - &(l_coll->buf_hndl), - node[dest_rank].r_done[alternate], - &(r_coll[dest_rank].buf_hndl)); -#endif - } - - - /* Wait till everyone is done with me */ - for (i = 0; i < size; i++) { - dest_rank = (rank + i) % size; - - if (dest_rank == rank) - continue; - - poll_addr_ptr = (void *) node[dest_rank].done[alternate]; - - while (*((volatile char *) poll_addr_ptr) == 0); - - /* Check to see if my sends to this process are complete */ - coll_all_send_complete(actual_comm->lrank_to_grank[dest_rank]); - } - - /* Deregister the buffers */ - dreg_unregister(d_recv); - dreg_unregister(d_send); - - return (MPI_SUCCESS); -} - -#endif /* VIADEV_RPUT_SUPPORT */ Modified: mvapich/trunk/src/context/Makefile.in =================================================================== --- mvapich/trunk/src/context/Makefile.in 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/context/Makefile.in 2008-11-24 04:12:11 UTC (rev 3143) @@ -36,7 +36,7 @@ comm_size.c comm_util.c comm_split.c context_util.c \ group_tranks.c comm_testic.c comm_rank.c commcompare.c \ comm_rsize.c comm_rgroup.c ic_create.c ic_merge.c \ - null_copyfn.c null_del_fn.c dup_fn.c comm_rdma_init.c create_2level_comm.c + null_copyfn.c null_del_fn.c dup_fn.c create_2level_comm.c MPI_TSOURCES = @@ -50,7 +50,7 @@ comm_size.o comm_util.o comm_split.o context_util.o \ group_tranks.o comm_testic.o commcompare.o \ comm_rsize.o comm_rgroup.o ic_create.o ic_merge.o \ - null_copyfn.o null_del_fn.o dup_fn.o comm_rdma_init.o create_2level_comm.o + null_copyfn.o null_del_fn.o dup_fn.o create_2level_comm.o default: ${libbuild_dir}/lib${MPILIBNAME}.a Modified: mvapich/trunk/src/context/comm_free.c =================================================================== --- mvapich/trunk/src/context/comm_free.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/context/comm_free.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -42,18 +42,6 @@ /* For MPIR_COLLOPS */ #include "mpicoll.h" -/* For free-ing collective support stuff */ -#if ((defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) && !defined(CH_GEN2_MRAIL)) -#ifdef USE_MRAIL -#include "../../mpid/vapi_multirail/collutils.h" -#elif defined(CH_GEN2) -#include "../../mpid/ch_gen2/collutils.h" -extern reg_entry* coll_reg; -#else -#include "../../mpid/vapi/collutils.h" -#endif -#endif - #define DBG(a) #define OUTFILE stdout @@ -61,7 +49,6 @@ int flag = 0; extern void free_2level_comm (struct MPIR_COMMUNICATOR*); #endif -extern int enable_rdma_collectives; #ifdef _SMP_ extern int enable_shmem_collectives; #endif @@ -143,47 +130,6 @@ (void)MPID_CommFree( comm ); - /* Free collective buffer resources */ -#if ((defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) && !defined (CH_GEN2_MRAIL)) - if (enable_rdma_collectives && !viadev_use_on_demand){ - /* Free only for collective communicator */ - if((comm->comm_coll == comm) && (comm->comm_type == MPIR_INTRA)) { - if(NULL != comm->collbuf) { - if(NULL != comm->collbuf->l_coll) { - if(NULL != comm->collbuf->l_coll->buf) { -#ifdef CH_GEN2 - deregister_memory((comm->collbuf->l_coll->mr)); - - /* togle the valid bit for this buffer*/ - coll_reg = viadev.coll_comm_reg; - while(coll_reg != NULL){ - if ((coll_reg->buf) == comm->collbuf->l_coll->buf){ - coll_reg->valid = 0; - } - coll_reg = coll_reg->next; - } - -#else - deregister_mem(&(comm->collbuf->l_coll->buf_hndl.hndl)); -#endif - free(comm->collbuf->l_coll->buf); - } - free(comm->collbuf->l_coll); - } - if(NULL != comm->collbuf->r_coll) { - free(comm->collbuf->r_coll); - } - if(NULL != comm->collbuf->atoa) { - if(NULL != comm->collbuf->atoa->de) { - free(comm->collbuf->atoa->de); - } - free(comm->collbuf->atoa); - } - } - } - } -#endif - #if (defined( _SMP_) && (defined(CH_GEN2) || defined(CH_SMP) || defined(CH_GEN2_UD) || defined(CH_PSM))) if((comm->comm_coll == comm) && (comm->comm_type == MPIR_INTRA) && (enable_shmem_collectives) && (!flag)) { free_2level_comm(comm); Deleted: mvapich/trunk/src/context/comm_rdma_init.c =================================================================== --- mvapich/trunk/src/context/comm_rdma_init.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/context/comm_rdma_init.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -1,411 +0,0 @@ - -/* - * Copyright (c) 2002-2008, The Ohio State University. All rights - * reserved. - * - * This file is part of the MVAPICH software package, developed by the - * team members of The Ohio State University's Network-Based Computing - * Laboratory (NBCL) and headed by Professor Dhabaleswar K. (DK) Panda. - * - * For detailed copyright and licensing information, please refer to the - * copright file COPYRIGHT_MVAPICH in the top level MPICH directory. - * - */ - - -#if ((defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT))) - -#include "mpiimpl.h" -#include "mpimem.h" -#include "mpid.h" -#include "mpiops.h" -#include "mpipt2pt.h" - -/* For device level initialization */ -#ifdef USE_MRAIL -#include "../../mpid/vapi_multirail/collutils.h" -#elif defined(CH_GEN2) -#include "../../mpid/ch_gen2/collutils.h" -reg_entry* coll_reg; -#elif (!defined(CH_GEN2_MRAIL)) -#include "../../mpid/vapi/collutils.h" -#endif - -/* Prototype for communicator RDMA features initialization*/ -void comm_rdma_init(struct MPIR_COMMUNICATOR*); -void comm_exch_addr(struct MPIR_COMMUNICATOR*); -void set_local_offsets(struct MPIR_COMMUNICATOR* comm); -void coll_prepare_flags(struct Collbuf*); -#ifdef _SMP_ -extern int disable_shmem_barrier; -#endif -void comm_rdma_init(struct MPIR_COMMUNICATOR* comm) -{ -#ifndef CH_GEN2_MRAIL - struct L_Collbuf* local_coll; - struct R_Collbuf* remote_coll; - - if(comm->np <= 1) { - /* We do not need to do any initialization */ - - comm->collbuf = NULL; -#if ((!defined(CH_GEN2_MRAIL)) && (!defined(USE_MRAIL)) ) - comm->is_allgather_enabled = 0; -#endif - comm->is_alltoall_enabled = 0; - comm->is_mcast_enabled = 0; - comm->is_barrier_enabled = 0; - return; - } - - comm->collbuf = NULL; - - char* value; - comm->is_alltoall_enabled = 0; - if ((value = getenv("VIADEV_USE_RDMA_ALLTOALL")) != NULL) { - comm->is_alltoall_enabled = atoi(value); - } - -#if ((!defined(CH_GEN2_MRAIL)) && (!defined(USE_MRAIL)) ) - - comm->is_allgather_enabled = 0; - if ((value = getenv("VIADEV_USE_RDMA_ALLGATHER")) != NULL) { - comm->is_allgather_enabled = atoi(value); - } -#endif - - if(NULL != getenv("DISABLE_HARDWARE_MCST")) { - /* The user has disabled Multicast */ - comm->is_mcast_enabled = 0; - } else { - /* The user has enabled Multicast */ - comm->is_mcast_enabled = 1; - } - - comm->is_barrier_enabled = 0; - if ((value = getenv("VIADEV_USE_RDMA_BARRIER")) != NULL) { - comm->is_barrier_enabled = atoi(value); - } - -#ifdef _SMP_ - if ((disable_shmem_barrier == 0) && (comm->is_barrier_enabled)){ - - printf("Only VIADEV_USE_SHMEM_BARRIER or VIADEV_USE_RDMA_BARRIER can be 1 at one time\n"); - MPI_Abort(comm->self,0); - } -#endif - - comm->collbuf = (struct Collbuf*) - malloc(sizeof(struct Collbuf)); - - CHECK_ALLOC(comm->collbuf); - - local_coll = (struct L_Collbuf*) - malloc(sizeof(struct L_Collbuf)); - - CHECK_ALLOC(local_coll); - - local_coll->buf = (void*) malloc(COLL_BUF_SIZE); - - CHECK_ALLOC(local_coll->buf); - - /* Set the collective buffer to zero */ - memset(local_coll->buf, 0, COLL_BUF_SIZE); - - /* Register the local memory */ -#ifdef CH_GEN2 - local_coll->mr = register_memory(local_coll->buf, COLL_BUF_SIZE); - - coll_reg = (reg_entry*) malloc(sizeof(reg_entry)); - coll_reg->buf = local_coll->buf; - coll_reg->valid = 1; - coll_reg->mem_hndl = local_coll->mr; - - if (viadev.coll_comm_reg == NULL){ - coll_reg->next = NULL; - viadev.coll_comm_reg = coll_reg; - } - else{ - coll_reg->next = viadev.coll_comm_reg; - viadev.coll_comm_reg = coll_reg; - } - -#else - register_mem(local_coll->buf, COLL_BUF_SIZE, - &(local_coll->buf_hndl)); -#endif - - comm->collbuf->l_coll = local_coll; - - /* Allocate space for - * storing remote buffer - * addresses */ - remote_coll = (struct R_Collbuf*) - malloc(sizeof(struct R_Collbuf) * comm->np); - - CHECK_ALLOC(remote_coll); - - /* Set all the memory to zero */ - memset(remote_coll, 0, sizeof(struct R_Collbuf) * comm->np); - - comm->collbuf->r_coll = remote_coll; - - /*Set the flags for rdma barrier */ - comm->collbuf->togle = 0; - comm->collbuf->rdma_barrier_id = 0; -#endif -} - -/* Exchange addresses with other processes */ -void comm_exch_addr(struct MPIR_COMMUNICATOR* comm) -{ -#ifndef CH_GEN2_MRAIL - int i = 0, right, left; - struct Coll_Addr_Exch send_pkt; - struct Coll_Addr_Exch *recv_pkt; - - MPI_Status* statarray = NULL; - MPI_Request* reqarray = NULL; - - if(comm->np <= 1) { - return; - } - - recv_pkt = (struct Coll_Addr_Exch*) - malloc(sizeof(struct Coll_Addr_Exch) * comm->np); - CHECK_ALLOC(recv_pkt); - - statarray = (MPI_Status*) malloc(2 * sizeof(MPI_Status) * comm->np); - CHECK_ALLOC(statarray); - - reqarray = (MPI_Request*) malloc(2 * sizeof(MPI_Request) * comm->np); - CHECK_ALLOC(reqarray); - - send_pkt.buf = comm->collbuf->l_coll->buf; -#ifdef CH_GEN2 - send_pkt.rkey = comm->collbuf->l_coll->mr->rkey; -#else - send_pkt.buf_hndl = comm->collbuf->l_coll->buf_hndl; -#endif - - right = (comm->local_rank + 1) % comm->np; - left = (comm->local_rank + comm->np - 1) % comm->np; - - for(i = 0; i < comm->np - 1; i++) { - - MPI_Sendrecv((void*)&send_pkt, sizeof(struct Coll_Addr_Exch), - MPI_BYTE, comm->lrank_to_grank[right], ADDR_EXCHANGE_TAG, - (void*)&(recv_pkt[left]),sizeof(struct Coll_Addr_Exch), - MPI_BYTE, comm->lrank_to_grank[left], ADDR_EXCHANGE_TAG, - MPI_COMM_WORLD, &(statarray[i])); - - right = (right+1) % comm->np; - left = (left + comm->np - 1) % comm->np; - } - - for(i = 0; i < comm->np; i++) { - comm->collbuf->r_coll[i].buf = recv_pkt[i].buf; -#ifdef CH_GEN2 - comm->collbuf->r_coll[i].rkey = recv_pkt[i].rkey; -#else - comm->collbuf->r_coll[i].buf_hndl.hndl = recv_pkt[i].buf_hndl.hndl; - comm->collbuf->r_coll[i].buf_hndl.lkey = recv_pkt[i].buf_hndl.lkey; - comm->collbuf->r_coll[i].buf_hndl.rkey = recv_pkt[i].buf_hndl.rkey; -#endif - } - - free(reqarray); - free(statarray); - free(recv_pkt); -#endif -} - -void set_local_offsets(struct MPIR_COMMUNICATOR* comm) -{ -#ifndef CH_GEN2_MRAIL - struct L_Collbuf* l_coll; - struct R_Collbuf* r_coll; - struct Alltoall* atoa = NULL; - struct Direct_Eager* de = NULL; - struct DE_Remote_Node* rnode = NULL; - int i; - - if(comm->np <= 1) { - return; - } - - l_coll = comm->collbuf->l_coll; - r_coll = comm->collbuf->r_coll; - - atoa = (struct Alltoall*) malloc(sizeof(struct Alltoall)); - CHECK_ALLOC(atoa); - - - /* Direct Eager stuff */ - de = (struct Direct_Eager*) malloc(sizeof(struct Direct_Eager)); - CHECK_ALLOC(de); - - rnode = (struct DE_Remote_Node*) malloc(sizeof(struct DE_Remote_Node) * - comm->np); - CHECK_ALLOC(rnode); - - for(i = 0; i < comm->np; i++) { - /* Don't set for myself */ - if ( comm->local_rank == i ) continue; - -#if defined(CH_GEN2) - rnode[i].recv_addr[0] = (void*) - ((char*)l_coll->buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + - (i* (sizeof(aint_t) + sizeof(uint32_t) + sizeof(char)))); -#else - rnode[i].recv_addr[0] = (void*) - ((char*)l_coll->buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + - (i* (sizeof(aint_t) + sizeof(VAPI_rkey_t) + sizeof(char)))); -#endif - - rnode[i].recv_addr[1] = (void*) - ((char*)rnode[i].recv_addr[0] + - (COLL_ADDR_RECV_SPACE/2)); - - rnode[i].done[0] = (void*) - ((char*)l_coll->buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + COLL_ADDR_RECV_SPACE + - i*sizeof(char)); - - rnode[i].done[1] = (void*) - ((char*)rnode[i].done[0] + (COLL_ACK_SPACE/2)); - -#if defined(CH_GEN2) - rnode[i].dest_addr[0] = (void*) - ((char*)r_coll[i].buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + - (comm->local_rank*(sizeof(aint_t) + sizeof(uint32_t) - + sizeof(char)))); -#else - rnode[i].dest_addr[0] = (void*) - ((char*)r_coll[i].buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + - (comm->local_rank*(sizeof(aint_t) + sizeof(VAPI_rkey_t) - + sizeof(char)))); -#endif - - rnode[i].dest_addr[1] = (void*) - ((char*)rnode[i].dest_addr[0] + - (COLL_ADDR_RECV_SPACE/2)); - - rnode[i].r_done[0] = (void*) - ((char*)r_coll[i].buf + COLL_FLAG_SPACE + - COLL_ADDR_SEND_SPACE + COLL_ADDR_RECV_SPACE + - (comm->local_rank*sizeof(char))); - - rnode[i].r_done[1] = (void*) - ((char*)rnode[i].r_done[0] + (COLL_ACK_SPACE/2)); - - } - - /* Start with the first set of buffers */ - de->alternate = 0; - - de->node = rnode; - - atoa->de = de; - - comm->collbuf->atoa = atoa; - - /*temp*/ - /* Set the Collective RDMA flag */ - coll_prepare_flags(comm->collbuf); -#endif -} - -/* A wrap around function to prepare the - * flags in the beginning of the collective - * buffer - */ -void coll_prepare_flags(struct Collbuf* coll) -{ -#ifndef CH_GEN2_MRAIL - /* Set the first byte to RDMA_FLAG */ - - *((char*)coll->l_coll->buf) = ALLTOALL_RDMA_FLAG; - - coll->atoa->rdma_flag = coll->l_coll->buf; - - /* Only a 1 byte long field */ - coll->atoa->rdma_flag_size = 1; -#endif -} - -#if ((!defined(CH_GEN2_MRAIL)) && (!defined(USE_MRAIL)) ) -void comm_rdma_allgather_init(struct MPIR_COMMUNICATOR *comm) -{ - struct Allgather *allgather; - int dimensions, i, j, mask, dest; - - struct L_Collbuf *l_coll; - struct R_Collbuf *r_coll; - - if(comm->np <= 1) { - return; - } - - l_coll = comm->collbuf->l_coll; - r_coll = comm->collbuf->r_coll; - - allgather = (struct Allgather *) malloc(sizeof(struct Allgather)); - CHECK_ALLOC(allgather); - - dimensions = log_2(comm->np); - allgather->dimensions = dimensions; - - allgather->peers = (struct Allgather_Peer *) - malloc(dimensions*sizeof(struct Allgather_Peer)); - CHECK_ALLOC(allgather->peers); - - - allgather->rdma_flag = (char *)((char*)l_coll->buf + ALLGATHER_OFFSET + ALLGATHER_COLL_BUF_SIZE - - COLL_RDMA_FLAG_SPACE); - *((char*)allgather->rdma_flag) = ALLGATHER_RDMA_FLAG; - allgather->rdma_flag_size = 1; - - for (mask=0x1,i=0; ilocal_rank ^ mask; - for (j=0; j<3; j++) { - allgather->peers[i].done[j] = - (char *)l_coll->buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE + i; - allgather->peers[i].r_done[j] = - (char *)r_coll[dest].buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE + i; -#ifdef CH_GEN2 - allgather->peers[i].recv_addr[j] = - (char *)l_coll->buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE - + COLL_FLAG_SPACE + i*(sizeof(uint32_t)+sizeof(aint_t)+sizeof(char)); - allgather->peers[i].dest_addr[j] = - (char *)r_coll[dest].buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE + - COLL_FLAG_SPACE + i*(sizeof(uint32_t)+sizeof(aint_t)+sizeof(char)); -#else - allgather->peers[i].recv_addr[j] = - (char *)l_coll->buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE - + COLL_FLAG_SPACE + i*(sizeof(VAPI_rkey_t)+sizeof(aint_t)+sizeof(char)); - allgather->peers[i].dest_addr[j] = - (char *)r_coll[dest].buf + j*COLL_SINGLE_BUF_SIZE + ALLGATHER_OFFSET + ALLGATHER_BUF_SIZE + - COLL_FLAG_SPACE + i*(sizeof(VAPI_rkey_t)+sizeof(aint_t)+sizeof(char)); -#endif - } - } - - /* This is to alternate b/w the back2back coll buffers */ - allgather->alternate = 0; - - comm->collbuf->allgather = allgather; - -#ifdef TIME_ALLGATHER - comm->collbuf->allgather->send_recv_time = 0; - comm->collbuf->allgather->reset_time = 0; - comm->collbuf->allgather->send_complete_time = 0; -#endif - -} -#endif -#endif /* VIADEV_RPUT_SUPPORT */ Modified: mvapich/trunk/src/context/comm_util.c =================================================================== --- mvapich/trunk/src/context/comm_util.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/context/comm_util.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -22,7 +22,6 @@ /* For MPIR_COLLOPS */ #include "mpicoll.h" -extern int enable_rdma_collectives; #ifdef _SMP_ extern int disable_shmem_allreduce; extern int enable_shmem_collectives; @@ -35,11 +34,9 @@ #endif #if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) -extern void comm_rdma_init(struct MPIR_COMMUNICATOR*); extern void comm_exch_addr(struct MPIR_COMMUNICATOR*); extern void set_local_offsets(struct MPIR_COMMUNICATOR* comm); extern void coll_prepare_flags(struct Collbuf*); -extern void comm_rdma_allgather_init(struct MPIR_COMMUNICATOR *); extern void clear_2level_comm (struct MPIR_COMMUNICATOR*); extern void create_2level_comm (struct MPIR_COMMUNICATOR*, int, int); #endif @@ -138,38 +135,6 @@ #endif -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) - /* Initializing communicator for Advanced collectives,doing it for only - INTRA */ - - /* - char* value; - if (((value = getenv("VIADEV_USE_RDMA_ALLTOALL")) != NULL)|| - ((value = getenv("VIADEV_USE_RDMA_ALLGATHER")) != NULL)|| - ((value = getenv("VIADEV_USE_RDMA_BARRIER")) != NULL)) { - enable_rdma_collectives = 1; - } - */ - - - if (enable_rdma_collectives && !viadev_use_on_demand){ - if (comm_type == MPIR_INTRA){ - - comm_rdma_init(new_comm); - comm_exch_addr(new_comm); -#if ((!(defined(CH_GEN2_MRAIL))) && (!(defined(USE_MRAIL))) ) - comm_rdma_allgather_init(new_comm); -#endif - set_local_offsets(new_comm); - - /* Initializing barrier info */ - - new_comm->rdma_barrier_id = 0; - new_comm->togle = 0; - - } - } -#endif /* VIADEV_RPUT_SUPPORT || VIADEV_RGET_SUPPORT */ #if (defined( _SMP_) && (defined(CH_GEN2) || defined(CH_SMP) || defined(CH_GEN2_UD) || defined(CH_PSM))) #if 1 if ((MPIR_Has_been_initialized == 1) && (enable_shmem_collectives)&&(comm_type == MPIR_INTRA)){ Modified: mvapich/trunk/src/env/initutil.c =================================================================== --- mvapich/trunk/src/env/initutil.c 2008-11-24 00:53:52 UTC (rev 3142) +++ mvapich/trunk/src/env/initutil.c 2008-11-24 04:12:11 UTC (rev 3143) @@ -47,20 +47,6 @@ #include "../../mpe/include/mpeexten.h" #endif -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) - -#if (!defined(CH_GEN2_MRAIL) && !defined(USE_MRAIL) ) - -#if (defined(CH_GEN2)) -#include "../../mpid/ch_gen2/viapriv.h" -#else -#include "../../mpid/vapi/viapriv.h" -#endif - -#endif /* !USE_MRAIL */ - -#endif /* VIADEV_RPUT_SUPPORT || VIADEV_RGET_SUPPORT */ - #ifndef PATCHLEVEL_SUBMINOR #define PATCHLEVEL_SUBMINOR 0 #endif @@ -174,7 +160,6 @@ extern long int allgather_large_msg_threshold; extern long int allgather_small_msg_threshold; int viadev_use_allgather_new =1; -int enable_rdma_collectives = 0; /* MPIR_Init - Initialize the MPI execution environment @@ -716,15 +701,6 @@ /* barrier */ MPIR_Has_been_initialized = 1; -#if (defined(VIADEV_RPUT_SUPPORT) || defined(VIADEV_RGET_SUPPORT)) - -#if (!defined(CH_GEN2_MRAIL) &&!defined(USE_MRAIL) ) - - viadev.initialized = 1; - -#endif /* !USE_MRAIL */ - -#endif /* VIADEV_RPUT_SUPPORT || VIADEV_RGET_SUPPORT */ #if (defined( _SMP_) && (defined(CH_GEN2) || defined(CH_SMP) || defined(CH_GEN2_UD) || defined(CH_PSM))) if (enable_shmem_collectives){ clear_2level_comm(MPIR_COMM_WORLD->comm_coll);