From vishnu at mvapich.cse.ohio-state.edu Tue May 8 21:17:16 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Tue May 8 21:17:30 2007 Subject: [mvapich-commit] r1219 - mvapich/trunk Message-ID: <200705090117.l491HGUn000484@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-08 21:17:16 -0400 (Tue, 08 May 2007) New Revision: 1219 Modified: mvapich/trunk/make.mvapich.gen2_multirail Log: Adding LAZY MEM UNREGISTER by default for multi-rail script Modified: mvapich/trunk/make.mvapich.gen2_multirail =================================================================== --- mvapich/trunk/make.mvapich.gen2_multirail 2007-05-09 01:15:41 UTC (rev 1218) +++ mvapich/trunk/make.mvapich.gen2_multirail 2007-05-09 01:17:16 UTC (rev 1219) @@ -66,7 +66,7 @@ export LIBS=${LIBS:--L${IBHOME_LIB} -Wl,-rpath=${IBHOME_LIB} -libverbs -libumad -lpthread} export FFLAGS=${FFLAGS:--L${IBHOME_LIB}} -export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} +export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DLAZY_MEM_UNREGISTER -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} # Prelogue make distclean &>/dev/null From mellanox at mvapich.cse.ohio-state.edu Sun May 13 10:12:11 2007 From: mellanox at mvapich.cse.ohio-state.edu (mellanox@mvapich.cse.ohio-state.edu) Date: Sun May 13 10:12:30 2007 Subject: [mvapich-commit] r1223 - in mvapich/branches/0.9.9: . mpid/ch_gen2 Message-ID: <200705131412.l4DECB5r011527@mvapich.cse.ohio-state.edu> Author: mellanox Date: 2007-05-13 10:12:07 -0400 (Sun, 13 May 2007) New Revision: 1223 Modified: mvapich/branches/0.9.9/mpid/ch_gen2/cm.c mvapich/branches/0.9.9/mpid/ch_gen2/ibverbs_const.h mvapich/branches/0.9.9/mpid/ch_gen2/viainit.c mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.c mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.h mvapich/branches/0.9.9/mvapich.conf Log: Fixing PKEY and PKEY INDEX support. OpenFabric bug - https://bugs.openfabrics.org/show_bug.cgi?id=519 Modified: mvapich/branches/0.9.9/mpid/ch_gen2/cm.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/cm.c 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mpid/ch_gen2/cm.c 2007-05-13 14:12:07 UTC (rev 1223) @@ -15,6 +15,7 @@ #include "viaparam.h" #include "viapriv.h" #include "cm_user.h" +#include "ib_init.h" typedef enum CM_conn_state_cli { CM_CONN_STATE_C_IDLE, @@ -877,8 +878,8 @@ memset(&attr, 0, sizeof(struct ibv_qp_attr)); attr.qp_state = IBV_QPS_INIT; - attr.pkey_index = viadev_default_pkey_ix; - attr.port_num = viadev_default_port; + attr.port_num = viadev_default_port; + set_pkey_index(&attr.pkey_index,viadev_default_port); attr.qkey = 0; if ((ret = ibv_modify_qp(cm_ud_qp, &attr, Modified: mvapich/branches/0.9.9/mpid/ch_gen2/ibverbs_const.h =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/ibverbs_const.h 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mpid/ch_gen2/ibverbs_const.h 2007-05-13 14:12:07 UTC (rev 1223) @@ -46,7 +46,7 @@ #define VIADEV_DEFAULT_PSN (0) #define VIADEV_DEFAULT_PKEY_IX (0) -#define VIADEV_DEFAULT_P_KEY (0x0) +#define VIADEV_DEFAULT_PKEY (0x0) #define VIADEV_DEFAULT_MIN_RNR_TIMER (12) #define VIADEV_DEFAULT_SERVICE_LEVEL (0) #define VIADEV_DEFAULT_TIME_OUT (20) Modified: mvapich/branches/0.9.9/mpid/ch_gen2/viainit.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/viainit.c 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mpid/ch_gen2/viainit.c 2007-05-13 14:12:07 UTC (rev 1223) @@ -60,6 +60,7 @@ #include "dreg.h" #include "process/pmgr_client.h" #include "mpid_smpi.h" +#include "ib_init.h" #ifndef DISABLE_PTMALLOC #include "mem_hooks.h" @@ -619,7 +620,7 @@ /*Init */ memset(&attr, 0, sizeof(struct ibv_qp_attr)); attr.qp_state = IBV_QPS_INIT; - attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); attr.port_num = viadev_default_port; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; @@ -1236,7 +1237,7 @@ } qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix ; + set_pkey_index(&qp_attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; @@ -2029,11 +2030,12 @@ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE; + if (ibv_modify_qp(viadev.boot_qp_hndl[0], &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | @@ -2345,7 +2347,7 @@ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qkey = 0; if (ret = ibv_modify_qp(viadev.ud_qp_hndl, &qp_attr, Modified: mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.c 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.c 2007-05-13 14:12:07 UTC (rev 1223) @@ -352,6 +352,7 @@ unsigned long viadev_max_rdma_size = VIADEV_MAX_RDMA_SIZE; uint8_t viadev_default_qp_ous_rd_atom = VIADEV_DEFAULT_QP_OUS_RD_ATOM; uint32_t viadev_default_psn = VIADEV_DEFAULT_PSN; +uint16_t viadev_default_pkey = VIADEV_DEFAULT_PKEY; uint16_t viadev_default_pkey_ix = VIADEV_DEFAULT_PKEY_IX; uint8_t viadev_default_min_rnr_timer = VIADEV_DEFAULT_MIN_RNR_TIMER; uint8_t viadev_default_service_level = VIADEV_DEFAULT_SERVICE_LEVEL; @@ -528,7 +529,10 @@ viadev_default_psn = (uint32_t)atol(value); } - if ((value = getenv("VIADEV_DEFAULT_PKEY_IX")) != NULL) { + if ((value = getenv("VIADEV_DEFAULT_PKEY")) != NULL) { + viadev_default_pkey = (uint16_t)strtol(value, (char **) NULL,0); + } + else if ((value = getenv("VIADEV_DEFAULT_PKEY_IX")) != NULL) { viadev_default_pkey_ix = (uint16_t)atol(value); } @@ -921,6 +925,7 @@ fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_MTU", (int)viadev_default_mtu); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_QP_OUS_RD_ATOM", (int)viadev_default_qp_ous_rd_atom); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PSN", (int)viadev_default_psn); + fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PKEY", (int)viadev_default_pkey); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PKEY_IX", (int)viadev_default_pkey_ix); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_MIN_RNR_TIMER", (int)viadev_default_min_rnr_timer); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_SERVICE_LEVEL", (int)viadev_default_service_level); Modified: mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.h =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.h 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mpid/ch_gen2/viaparam.h 2007-05-13 14:12:07 UTC (rev 1223) @@ -73,6 +73,7 @@ extern unsigned long viadev_max_rdma_size; extern uint8_t viadev_default_qp_ous_rd_atom; extern uint32_t viadev_default_psn; +extern uint16_t viadev_default_pkey; extern uint16_t viadev_default_pkey_ix; extern uint8_t viadev_default_min_rnr_timer; extern uint8_t viadev_default_service_level; Modified: mvapich/branches/0.9.9/mvapich.conf =================================================================== --- mvapich/branches/0.9.9/mvapich.conf 2007-05-11 03:02:16 UTC (rev 1222) +++ mvapich/branches/0.9.9/mvapich.conf 2007-05-13 14:12:07 UTC (rev 1223) @@ -552,7 +552,7 @@ # a packet is dropped and retrying. Increasing values # increase the amount of time. #----------------------------------------------------------------------------------------- -# VIADEV_CQ_SIZE +# VIADEV_CQ_SIZE=40000 # # Default value:40000 # Number of completion queue entries. @@ -561,14 +561,24 @@ # InfiniBand layer used by MVAPICH communication. # # VIADEV_DEFAULT_PSN -# VIADEV_DEFAULT_PKEY_IX # VIADEV_DEFAULT_SERVICE_LEVEL # VIADEV_DEFAULT_STATIC_RATE # VIADEV_DEFAULT_SRC_PATH_BITS # VIADEV_DEFAULT_MAX_SG_LIST # VIADEV_DEFAULT_MAX_RDMA_DST_OPS #----------------------------------------------------------------------------------------- +# VIADEV_DEFAULT_PKEY_IX # +# Default value:none +# Index of predefined partition +#----------------------------------------------------------------------------------------- +# VIADEV_DEFAULT_PKEY +# +# Default value:none +# Name (key) of predefined partition. One can pass it both in decimal end hexadecimal +# format. This parameter prevail over VIADEV_DEFAULT_PKEY_IX, when defined. +#----------------------------------------------------------------------------------------- +# # 9. Multi-Rail Device Control Parameters # #----------------------------------------------------------------------------------------- From mellanox at mvapich.cse.ohio-state.edu Mon May 14 04:45:50 2007 From: mellanox at mvapich.cse.ohio-state.edu (mellanox@mvapich.cse.ohio-state.edu) Date: Mon May 14 04:46:08 2007 Subject: [mvapich-commit] r1225 - mvapich/branches/0.9.9/mpid/ch_gen2 Message-ID: <200705140845.l4E8joaH013197@mvapich.cse.ohio-state.edu> Author: mellanox Date: 2007-05-14 04:45:49 -0400 (Mon, 14 May 2007) New Revision: 1225 Added: mvapich/branches/0.9.9/mpid/ch_gen2/ib_init.h Log: Adding ib_init.h that I forgot to add in previous commit (PKEY support r1223) Added: mvapich/branches/0.9.9/mpid/ch_gen2/ib_init.h =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/ib_init.h 2007-05-13 19:20:21 UTC (rev 1224) +++ mvapich/branches/0.9.9/mpid/ch_gen2/ib_init.h 2007-05-14 08:45:49 UTC (rev 1225) @@ -0,0 +1,39 @@ +#ifndef IB_INIT_H +#define IB_INIT_H + +#include "viaparam.h" +#include "ibverbs_const.h" +#include "viadev.h" +/* +This functions locates PKEY INDEX by PKEY itself +It returns PKEY in the case of success, or int bad_pkey_idx otherwise +*/ +static inline uint16_t get_pkey_index(uint16_t pkey, int port_num) { + static const uint16_t bad_pkey_idx = -1; + uint16_t i; + if(ibv_query_device(viadev.context, &viadev.dev_attr)) { + error_abort_all(GEN_EXIT_ERR, + "Error getting HCA attributes\n"); + } + for (i = 0; i < viadev.dev_attr.max_pkeys ; ++i) { + uint16_t curr_pkey; + ibv_query_pkey(viadev.context, (uint8_t)port_num, (int)i ,&curr_pkey); + if (pkey == ntohs(curr_pkey)) { + return i; + } + } + return bad_pkey_idx; +} + +/* +This functions sets PKEY INDEX according to PKEY, if PKEY was defined by user. +*/ + +static inline void set_pkey_index(uint16_t * pkey_index, int port_num) { + *pkey_index = (viadev_default_pkey == VIADEV_DEFAULT_PKEY ? viadev_default_pkey_ix : get_pkey_index(viadev_default_pkey,port_num)); + if (pkey_index < 0 ) { + error_abort_all(IBV_RETURN_ERR, + "Can't find PKEY INDEX according to given PKEY\n"); + } +} +#endif //IB_INIT_H From vishnu at mvapich.cse.ohio-state.edu Mon May 14 12:36:39 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Mon May 14 12:36:56 2007 Subject: [mvapich-commit] r1226 - mvapich/branches/0.9.9 Message-ID: <200705141636.l4EGac2w014036@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-14 12:36:36 -0400 (Mon, 14 May 2007) New Revision: 1226 Modified: mvapich/branches/0.9.9/make.mvapich.gen2_multirail Log: Adding the LAZY_MEM_UNREGISTER to the compilation script for gen2_multirail Modified: mvapich/branches/0.9.9/make.mvapich.gen2_multirail =================================================================== --- mvapich/branches/0.9.9/make.mvapich.gen2_multirail 2007-05-14 08:45:49 UTC (rev 1225) +++ mvapich/branches/0.9.9/make.mvapich.gen2_multirail 2007-05-14 16:36:36 UTC (rev 1226) @@ -66,7 +66,7 @@ export LIBS=${LIBS:--L${IBHOME_LIB} -Wl,-rpath=${IBHOME_LIB} -libverbs -libumad -lpthread} export FFLAGS=${FFLAGS:--L${IBHOME_LIB}} -export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} +export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DLAZY_MEM_UNREGISTER -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} # Prelogue make distclean &>/dev/null From vishnu at mvapich.cse.ohio-state.edu Mon May 14 12:50:07 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Mon May 14 12:50:23 2007 Subject: [mvapich-commit] r1227 - mvapich/branches/0.9.9/mpid/ch_gen2_multirail Message-ID: <200705141650.l4EGo7iW014065@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-14 12:50:06 -0400 (Mon, 14 May 2007) New Revision: 1227 Modified: mvapich/branches/0.9.9/mpid/ch_gen2_multirail/Makefile.in mvapich/branches/0.9.9/mpid/ch_gen2_multirail/vbuf.c mvapich/branches/0.9.9/mpid/ch_gen2_multirail/viapriv.c Log: sync'ing the multi-rail trunk version to 0.9.9 1. Changes to the Makefile.in for CFLAGS 2. By default, the number of qps/port used to be 1 Modified: mvapich/branches/0.9.9/mpid/ch_gen2_multirail/Makefile.in =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2_multirail/Makefile.in 2007-05-14 16:36:36 UTC (rev 1226) +++ mvapich/branches/0.9.9/mpid/ch_gen2_multirail/Makefile.in 2007-05-14 16:50:06 UTC (rev 1227) @@ -81,7 +81,7 @@ # default_all is the target used by the MPICH build. It can be optimized # to not to the ranlib that default does. Is this necessary on modern machines? -MPIRUN_CFLAGS = -DVAPI @CC_SHARED_OPT@ +MPIRUN_CFLAGS += -DVAPI @CC_SHARED_OPT@ ifeq "$(findstring -DUSE_MPD,$(CFLAGS1))" "-DUSE_MPD" ifeq "$(findstring -DUSE_MPD_RING,$(CFLAGS1))" "-DUSE_MPD_RING" Modified: mvapich/branches/0.9.9/mpid/ch_gen2_multirail/vbuf.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2_multirail/vbuf.c 2007-05-14 16:36:36 UTC (rev 1226) +++ mvapich/branches/0.9.9/mpid/ch_gen2_multirail/vbuf.c 2007-05-14 16:50:06 UTC (rev 1227) @@ -70,13 +70,13 @@ for(j = 0; j < num_hcas; j++) { reg->mem_handle[j] = register_memory(free_vbuf_head, nvbufs * sizeof(vbuf), j); - if (NULL == reg->mem_handle[j]) { error_abort_all(GEN_EXIT_ERR, "unable to register vbuf DMA buffer"); } } - + + /* init the free list */ for (i = 0; i < nvbufs - 1; i++) { cur = free_vbuf_head + i; Modified: mvapich/branches/0.9.9/mpid/ch_gen2_multirail/viapriv.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2_multirail/viapriv.c 2007-05-14 16:36:36 UTC (rev 1226) +++ mvapich/branches/0.9.9/mpid/ch_gen2_multirail/viapriv.c 2007-05-14 16:50:06 UTC (rev 1227) @@ -31,9 +31,9 @@ int num_ports = 1; int num_hcas = 1; -int num_subchannels = 2; +int num_subchannels = 1; int num_phy_ports = 1; -int num_qp_per_port = 2; +int num_qp_per_port = 1; int backlog_counter = 0; int link_speed[MAX_SUBCHANNELS]; From surs at mvapich.cse.ohio-state.edu Mon May 14 13:41:59 2007 From: surs at mvapich.cse.ohio-state.edu (surs@mvapich.cse.ohio-state.edu) Date: Mon May 14 13:42:18 2007 Subject: [mvapich-commit] r1228 - in mvapich/trunk: . mpid/ch_gen2 src/env Message-ID: <200705141741.l4EHfxPH014150@mvapich.cse.ohio-state.edu> Author: surs Date: 2007-05-14 13:41:58 -0400 (Mon, 14 May 2007) New Revision: 1228 Added: mvapich/trunk/mpid/ch_gen2/ib_init.h Modified: mvapich/trunk/CHANGELOG mvapich/trunk/make.mvapich.gen2_multirail mvapich/trunk/mpid/ch_gen2/cm.c mvapich/trunk/mpid/ch_gen2/ibverbs_const.h mvapich/trunk/mpid/ch_gen2/viainit.c mvapich/trunk/mpid/ch_gen2/viaparam.c mvapich/trunk/mpid/ch_gen2/viaparam.h mvapich/trunk/mvapich.conf mvapich/trunk/src/env/initutil.c Log: -- Sync from 0.9.9 to trunk Modified: mvapich/trunk/CHANGELOG =================================================================== --- mvapich/trunk/CHANGELOG 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/CHANGELOG 2007-05-14 17:41:58 UTC (rev 1228) @@ -6,7 +6,7 @@ 04/25/2007 -* Changed shmem collectives macros tunable at run-time +* Made shared memory macros tunable at run time 04/24/2007 Modified: mvapich/trunk/make.mvapich.gen2_multirail =================================================================== --- mvapich/trunk/make.mvapich.gen2_multirail 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/make.mvapich.gen2_multirail 2007-05-14 17:41:58 UTC (rev 1228) @@ -66,7 +66,7 @@ export LIBS=${LIBS:--L${IBHOME_LIB} -Wl,-rpath=${IBHOME_LIB} -libverbs -libumad -lpthread} export FFLAGS=${FFLAGS:--L${IBHOME_LIB}} -export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DLAZY_MEM_UNREGISTER -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} +export CFLAGS=${CFLAGS:--D${ARCH} -DRDMA_FAST_PATH -DLAZY_MEM_UNREGISTER -DEARLY_SEND_COMPLETION -DVIADEV_RPUT_SUPPORT -D_SMP_ -D_SMP_RNDV_ -DCH_GEN2_MRAIL ${COMPILER_FLAG} -I${IBHOME}/include $OPT_FLAG} # Prelogue make distclean &>/dev/null Modified: mvapich/trunk/mpid/ch_gen2/cm.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/cm.c 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/cm.c 2007-05-14 17:41:58 UTC (rev 1228) @@ -15,6 +15,7 @@ #include "viaparam.h" #include "viapriv.h" #include "cm_user.h" +#include "ib_init.h" typedef enum CM_conn_state_cli { CM_CONN_STATE_C_IDLE, @@ -877,8 +878,8 @@ memset(&attr, 0, sizeof(struct ibv_qp_attr)); attr.qp_state = IBV_QPS_INIT; - attr.pkey_index = viadev_default_pkey_ix; - attr.port_num = viadev_default_port; + attr.port_num = viadev_default_port; + set_pkey_index(&attr.pkey_index,viadev_default_port); attr.qkey = 0; if ((ret = ibv_modify_qp(cm_ud_qp, &attr, Added: mvapich/trunk/mpid/ch_gen2/ib_init.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/ib_init.h 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/ib_init.h 2007-05-14 17:41:58 UTC (rev 1228) @@ -0,0 +1,39 @@ +#ifndef IB_INIT_H +#define IB_INIT_H + +#include "viaparam.h" +#include "ibverbs_const.h" +#include "viadev.h" +/* +This functions locates PKEY INDEX by PKEY itself +It returns PKEY in the case of success, or int bad_pkey_idx otherwise +*/ +static inline uint16_t get_pkey_index(uint16_t pkey, int port_num) { + static const uint16_t bad_pkey_idx = -1; + uint16_t i; + if(ibv_query_device(viadev.context, &viadev.dev_attr)) { + error_abort_all(GEN_EXIT_ERR, + "Error getting HCA attributes\n"); + } + for (i = 0; i < viadev.dev_attr.max_pkeys ; ++i) { + uint16_t curr_pkey; + ibv_query_pkey(viadev.context, (uint8_t)port_num, (int)i ,&curr_pkey); + if (pkey == ntohs(curr_pkey)) { + return i; + } + } + return bad_pkey_idx; +} + +/* +This functions sets PKEY INDEX according to PKEY, if PKEY was defined by user. +*/ + +static inline void set_pkey_index(uint16_t * pkey_index, int port_num) { + *pkey_index = (viadev_default_pkey == VIADEV_DEFAULT_PKEY ? viadev_default_pkey_ix : get_pkey_index(viadev_default_pkey,port_num)); + if (pkey_index < 0 ) { + error_abort_all(IBV_RETURN_ERR, + "Can't find PKEY INDEX according to given PKEY\n"); + } +} +#endif //IB_INIT_H Modified: mvapich/trunk/mpid/ch_gen2/ibverbs_const.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/ibverbs_const.h 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/ibverbs_const.h 2007-05-14 17:41:58 UTC (rev 1228) @@ -46,7 +46,7 @@ #define VIADEV_DEFAULT_PSN (0) #define VIADEV_DEFAULT_PKEY_IX (0) -#define VIADEV_DEFAULT_P_KEY (0x0) +#define VIADEV_DEFAULT_PKEY (0x0) #define VIADEV_DEFAULT_MIN_RNR_TIMER (12) #define VIADEV_DEFAULT_SERVICE_LEVEL (0) #define VIADEV_DEFAULT_TIME_OUT (20) Modified: mvapich/trunk/mpid/ch_gen2/viainit.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viainit.c 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/viainit.c 2007-05-14 17:41:58 UTC (rev 1228) @@ -60,6 +60,7 @@ #include "dreg.h" #include "process/pmgr_client.h" #include "mpid_smpi.h" +#include "ib_init.h" #ifndef DISABLE_PTMALLOC #include "mem_hooks.h" @@ -619,7 +620,7 @@ /*Init */ memset(&attr, 0, sizeof(struct ibv_qp_attr)); attr.qp_state = IBV_QPS_INIT; - attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); attr.port_num = viadev_default_port; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; @@ -1236,7 +1237,7 @@ } qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix ; + set_pkey_index(&qp_attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; @@ -2029,11 +2030,12 @@ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE; + if (ibv_modify_qp(viadev.boot_qp_hndl[0], &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | @@ -2345,7 +2347,7 @@ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = viadev_default_pkey_ix; + set_pkey_index(&attr.pkey_index,viadev_default_port); qp_attr.port_num = viadev_default_port; qp_attr.qkey = 0; if (ret = ibv_modify_qp(viadev.ud_qp_hndl, &qp_attr, Modified: mvapich/trunk/mpid/ch_gen2/viaparam.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-05-14 17:41:58 UTC (rev 1228) @@ -352,6 +352,7 @@ unsigned long viadev_max_rdma_size = VIADEV_MAX_RDMA_SIZE; uint8_t viadev_default_qp_ous_rd_atom = VIADEV_DEFAULT_QP_OUS_RD_ATOM; uint32_t viadev_default_psn = VIADEV_DEFAULT_PSN; +uint16_t viadev_default_pkey = VIADEV_DEFAULT_PKEY; uint16_t viadev_default_pkey_ix = VIADEV_DEFAULT_PKEY_IX; uint8_t viadev_default_min_rnr_timer = VIADEV_DEFAULT_MIN_RNR_TIMER; uint8_t viadev_default_service_level = VIADEV_DEFAULT_SERVICE_LEVEL; @@ -528,7 +529,10 @@ viadev_default_psn = (uint32_t)atol(value); } - if ((value = getenv("VIADEV_DEFAULT_PKEY_IX")) != NULL) { + if ((value = getenv("VIADEV_DEFAULT_PKEY")) != NULL) { + viadev_default_pkey = (uint16_t)strtol(value, (char **) NULL,0); + } + else if ((value = getenv("VIADEV_DEFAULT_PKEY_IX")) != NULL) { viadev_default_pkey_ix = (uint16_t)atol(value); } @@ -921,6 +925,7 @@ fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_MTU", (int)viadev_default_mtu); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_QP_OUS_RD_ATOM", (int)viadev_default_qp_ous_rd_atom); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PSN", (int)viadev_default_psn); + fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PKEY", (int)viadev_default_pkey); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_PKEY_IX", (int)viadev_default_pkey_ix); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_MIN_RNR_TIMER", (int)viadev_default_min_rnr_timer); fprintf(fd, int_fmt, me, "VIADEV_DEFAULT_SERVICE_LEVEL", (int)viadev_default_service_level); Modified: mvapich/trunk/mpid/ch_gen2/viaparam.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/viaparam.h 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mpid/ch_gen2/viaparam.h 2007-05-14 17:41:58 UTC (rev 1228) @@ -73,6 +73,7 @@ extern unsigned long viadev_max_rdma_size; extern uint8_t viadev_default_qp_ous_rd_atom; extern uint32_t viadev_default_psn; +extern uint16_t viadev_default_pkey; extern uint16_t viadev_default_pkey_ix; extern uint8_t viadev_default_min_rnr_timer; extern uint8_t viadev_default_service_level; Modified: mvapich/trunk/mvapich.conf =================================================================== --- mvapich/trunk/mvapich.conf 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/mvapich.conf 2007-05-14 17:41:58 UTC (rev 1228) @@ -552,7 +552,7 @@ # a packet is dropped and retrying. Increasing values # increase the amount of time. #----------------------------------------------------------------------------------------- -# VIADEV_CQ_SIZE +# VIADEV_CQ_SIZE=40000 # # Default value:40000 # Number of completion queue entries. @@ -561,14 +561,24 @@ # InfiniBand layer used by MVAPICH communication. # # VIADEV_DEFAULT_PSN -# VIADEV_DEFAULT_PKEY_IX # VIADEV_DEFAULT_SERVICE_LEVEL # VIADEV_DEFAULT_STATIC_RATE # VIADEV_DEFAULT_SRC_PATH_BITS # VIADEV_DEFAULT_MAX_SG_LIST # VIADEV_DEFAULT_MAX_RDMA_DST_OPS #----------------------------------------------------------------------------------------- +# VIADEV_DEFAULT_PKEY_IX # +# Default value:none +# Index of predefined partition +#----------------------------------------------------------------------------------------- +# VIADEV_DEFAULT_PKEY +# +# Default value:none +# Name (key) of predefined partition. One can pass it both in decimal end hexadecimal +# format. This parameter prevail over VIADEV_DEFAULT_PKEY_IX, when defined. +#----------------------------------------------------------------------------------------- +# # 9. Multi-Rail Device Control Parameters # #----------------------------------------------------------------------------------------- Modified: mvapich/trunk/src/env/initutil.c =================================================================== --- mvapich/trunk/src/env/initutil.c 2007-05-14 16:50:06 UTC (rev 1227) +++ mvapich/trunk/src/env/initutil.c 2007-05-14 17:41:58 UTC (rev 1228) @@ -148,7 +148,7 @@ extern int disable_shmem_allreduce; int split_comm = 1; extern int shmem_coll_blocks; -#if (defined(CH_GEN2)) || (defined(CH_SMP)) +#if (defined (CH_GEN2) || defined (CH_SMP)) extern int shmem_coll_max_msg_size; extern int shmem_coll_reduce_threshold; extern int shmem_coll_allreduce_threshold; From vishnu at mvapich.cse.ohio-state.edu Thu May 17 11:52:54 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Thu May 17 11:53:11 2007 Subject: [mvapich-commit] r1232 - mvapich/trunk/mpid/ch_smp/process Message-ID: <200705171552.l4HFqstA021449@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-17 11:52:52 -0400 (Thu, 17 May 2007) New Revision: 1232 Modified: mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c Log: Checking in the correct string for the SMP device, to be displayed with -v option of mpirun_rsh Modified: mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c 2007-05-16 20:29:47 UTC (rev 1231) +++ mvapich/trunk/mpid/ch_smp/process/mpirun_rsh.c 2007-05-17 15:52:52 UTC (rev 1232) @@ -180,7 +180,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-SingleRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-SMP\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } From vishnu at mvapich.cse.ohio-state.edu Thu May 17 11:56:04 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Thu May 17 11:56:20 2007 Subject: [mvapich-commit] r1233 - mvapich/trunk/mpid/vapi/process Message-ID: <200705171556.l4HFu44u021468@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-17 11:56:02 -0400 (Thu, 17 May 2007) New Revision: 1233 Modified: mvapich/trunk/mpid/vapi/process/mpirun_rsh.c Log: Adding VAPI string to differentiate between VAPI and Gen2 singlerail device Modified: mvapich/trunk/mpid/vapi/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/vapi/process/mpirun_rsh.c 2007-05-17 15:52:52 UTC (rev 1232) +++ mvapich/trunk/mpid/vapi/process/mpirun_rsh.c 2007-05-17 15:56:02 UTC (rev 1233) @@ -172,7 +172,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-SingleRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-VAPI-SingleRail\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } From vishnu at mvapich.cse.ohio-state.edu Thu May 17 11:57:56 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Thu May 17 11:58:12 2007 Subject: [mvapich-commit] r1234 - mvapich/trunk/mpid/vapi_multirail/process Message-ID: <200705171557.l4HFvubf021484@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-17 11:57:56 -0400 (Thu, 17 May 2007) New Revision: 1234 Modified: mvapich/trunk/mpid/vapi_multirail/process/mpirun_rsh.c Log: Adding VAPI string to differentiate between VAPI and Gen2 multirail device Modified: mvapich/trunk/mpid/vapi_multirail/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/vapi_multirail/process/mpirun_rsh.c 2007-05-17 15:56:02 UTC (rev 1233) +++ mvapich/trunk/mpid/vapi_multirail/process/mpirun_rsh.c 2007-05-17 15:57:56 UTC (rev 1234) @@ -167,7 +167,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-MultiRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-VAPI-MultiRail\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } From vishnu at mvapich.cse.ohio-state.edu Thu May 17 12:34:24 2007 From: vishnu at mvapich.cse.ohio-state.edu (vishnu@mvapich.cse.ohio-state.edu) Date: Thu May 17 12:34:43 2007 Subject: [mvapich-commit] r1238 - in mvapich/branches/0.9.9/mpid: ch_smp/process vapi/process vapi_multirail/process Message-ID: <200705171634.l4HGYOKe021603@mvapich.cse.ohio-state.edu> Author: vishnu Date: 2007-05-17 12:34:22 -0400 (Thu, 17 May 2007) New Revision: 1238 Modified: mvapich/branches/0.9.9/mpid/ch_smp/process/mpirun_rsh.c mvapich/branches/0.9.9/mpid/vapi/process/mpirun_rsh.c mvapich/branches/0.9.9/mpid/vapi_multirail/process/mpirun_rsh.c Log: Adding the strings for mpirun_rsh -v, corresponding to the changes in the trunk. The changes are for ch_smp, vapi, and vapi_multirail device Modified: mvapich/branches/0.9.9/mpid/ch_smp/process/mpirun_rsh.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_smp/process/mpirun_rsh.c 2007-05-17 16:21:32 UTC (rev 1237) +++ mvapich/branches/0.9.9/mpid/ch_smp/process/mpirun_rsh.c 2007-05-17 16:34:22 UTC (rev 1238) @@ -180,7 +180,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-SingleRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-SMP\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } Modified: mvapich/branches/0.9.9/mpid/vapi/process/mpirun_rsh.c =================================================================== --- mvapich/branches/0.9.9/mpid/vapi/process/mpirun_rsh.c 2007-05-17 16:21:32 UTC (rev 1237) +++ mvapich/branches/0.9.9/mpid/vapi/process/mpirun_rsh.c 2007-05-17 16:34:22 UTC (rev 1238) @@ -172,7 +172,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-SingleRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-VAPI-SingleRail\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } Modified: mvapich/branches/0.9.9/mpid/vapi_multirail/process/mpirun_rsh.c =================================================================== --- mvapich/branches/0.9.9/mpid/vapi_multirail/process/mpirun_rsh.c 2007-05-17 16:21:32 UTC (rev 1237) +++ mvapich/branches/0.9.9/mpid/vapi_multirail/process/mpirun_rsh.c 2007-05-17 16:34:22 UTC (rev 1238) @@ -167,7 +167,7 @@ static void show_version(void) { - fprintf(stderr,"OSU MVAPICH VERSION %s-MultiRail\n" + fprintf(stderr,"OSU MVAPICH VERSION %s-VAPI-MultiRail\n" "Build-ID: %s\n", MVAPICH_VERSION, MVAPICH_BUILDID); } From mellanox at mvapich.cse.ohio-state.edu Tue May 29 03:47:12 2007 From: mellanox at mvapich.cse.ohio-state.edu (mellanox@mvapich.cse.ohio-state.edu) Date: Tue May 29 03:47:27 2007 Subject: [mvapich-commit] r1283 - in mvapich/branches/0.9.9/mpid/ch_gen2: . process Message-ID: <200705290747.l4T7lCvs031538@mvapich.cse.ohio-state.edu> Author: mellanox Date: 2007-05-29 03:47:10 -0400 (Tue, 29 May 2007) New Revision: 1283 Modified: mvapich/branches/0.9.9/mpid/ch_gen2/mpid_init.c mvapich/branches/0.9.9/mpid/ch_gen2/process/mpirun_rsh.c mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client.h mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpd.c mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpirun_rsh.c mvapich/branches/0.9.9/mpid/ch_gen2/viacheck.c mvapich/branches/0.9.9/mpid/ch_gen2/viutil.h Log: Improving error_abort_all report. Now it will report source and destination of failed packets. Modified: mvapich/branches/0.9.9/mpid/ch_gen2/mpid_init.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/mpid_init.c 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/mpid_init.c 2007-05-29 07:47:10 UTC (rev 1283) @@ -93,7 +93,7 @@ MPIR_debug_state = MPIR_DEBUG_ABORTING; MPIR_Breakpoint(); - pmgr_abort(); + pmgr_abort(code); exit(code); } Modified: mvapich/branches/0.9.9/mpid/ch_gen2/process/mpirun_rsh.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/process/mpirun_rsh.c 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/process/mpirun_rsh.c 2007-05-29 07:47:10 UTC (rev 1283) @@ -1019,26 +1019,40 @@ } void wait_for_errors(int s,struct sockaddr_in *sockaddr,unsigned int sockaddr_len){ - - int nread,remote_id,s1,i; - + + int nread,remote_id,local_id,s1,i,flag; + s1 = accept(s,(struct sockaddr *) sockaddr,&sockaddr_len); - nread = read(s1, &remote_id, sizeof(remote_id)); + nread = read(s1, &flag, sizeof(flag)); if (nread == -1) { perror("Termination socket read failed"); } else if (nread == 0) { - } else if (nread != sizeof(remote_id)) { + } else if (nread != sizeof(flag)) { printf("Invalid termination socket on read\n"); cleanup(); } else { - printf("mpirun_rsh: Abort signaled from [%d]\n",remote_id); - /*if (remote_id == ABORT_ERROR) { - + printf("Aborting code !\n"); - + */ - /* shut down all our ports */ - close(s); - close(s1); - cleanup(); + nread = read(s1, &local_id, sizeof(local_id)); + if (nread == -1) { + perror("Termination socket read failed"); + } else if (nread == 0) { + } else if (nread != sizeof(local_id)) { + printf("Invalid termination socket on read\n"); + cleanup(); + } else if (flag > -1) { + remote_id=flag; + printf("mpirun_rsh: Abort signaled from [%d : %s] remote host is [%d : %s ]\n",local_id,plist[local_id].hostname,remote_id, plist[remote_id].hostname); + close(s); + close(s1); + cleanup(); + } + else + { + printf("mpirun_rsh: Abort signaled from [%d]\n",local_id); + close(s); + close(s1); + cleanup(); + + } } } Modified: mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client.h =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client.h 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client.h 2007-05-29 07:47:10 UTC (rev 1283) @@ -93,7 +93,7 @@ */ int pmgr_finalize(void); -int pmgr_abort(void); +int pmgr_abort(int); int pmgr_get_mpirun_process(int np,char ***processes_p); #ifdef MCST_SUPPORT Modified: mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpd.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpd.c 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpd.c 2007-05-29 07:47:10 UTC (rev 1283) @@ -315,7 +315,7 @@ /* abort call to process manager. Allows it to clean-up * any resources it might have allocated. */ -int pmgr_abort() +int pmgr_abort(int none) { MPD_Abort(1); return(1); Modified: mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpirun_rsh.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpirun_rsh.c 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/process/pmgr_client_mpirun_rsh.c 2007-05-29 07:47:10 UTC (rev 1283) @@ -439,11 +439,12 @@ * Call into the process spawner, using the same port we were given * at startup time, to tell it to abort the entire job. */ -int pmgr_abort(void) +int pmgr_abort(int flag) { int s; struct sockaddr_in sin; struct hostent *he; + char* str; he = gethostbyname(mpirun_hostname); if (!he) { @@ -466,7 +467,7 @@ /* write our rank to mpirun_rsh (wait_for_error) * for use in nice error messages */ - + write(s, &flag, sizeof(flag)); write(s, &pmgr_me, sizeof(pmgr_me)); close(s); Modified: mvapich/branches/0.9.9/mpid/ch_gen2/viacheck.c =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/viacheck.c 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/viacheck.c 2007-05-29 07:47:10 UTC (rev 1283) @@ -325,7 +325,7 @@ if (ret2 == 1) { vbuf_addr = (void *) ((aint_t) sc.wr_id); if (sc.status != IBV_WC_SUCCESS) { - error_abort_all(IBV_STATUS_ERR, + error_abort_all(((vbuf *) vbuf_addr)->grank, "[%s:%d] Got completion with error %s, " "code=%d, dest rank=%d\n", viadev.my_name, viadev.me, @@ -340,7 +340,7 @@ if (ret3 == 1) { vbuf_addr = (void *) ((aint_t) sc.wr_id); if (sc.status != IBV_WC_SUCCESS) { - error_abort_all(IBV_STATUS_ERR, + error_abort_all(((vbuf *) vbuf_addr)->grank, "[%s:%d] Got completion with error %s, " "code=%d, dest rank=%d\n", viadev.my_name, viadev.me, @@ -385,7 +385,7 @@ /* Need to check if it is a completion with error */ if (sc.status != IBV_WC_SUCCESS) { - error_abort_all(IBV_STATUS_ERR, + error_abort_all(((vbuf *) vbuf_addr)->grank, "[%s:%d] Got completion with error %s, " "code=%d, dest rank=%d\n", viadev.my_name, viadev.me, @@ -2365,13 +2365,14 @@ } if (ne > 0) { - + void * vbuf_addr = (void *) ((aint_t) sc->wr_id); if (sc->status != IBV_WC_SUCCESS) { - error_abort_all(IBV_STATUS_ERR, + error_abort_all(((vbuf *) vbuf_addr)->grank, "[%s:%d] Got completion with error %s, " - "code=%d\n", viadev.my_name, - viadev.me, wc_code_to_str(sc->status), - sc->status); + "code=%d, dest rank=%d\n", + viadev.my_name, viadev.me, + wc_code_to_str(sc->status), sc->status, + ((vbuf *) vbuf_addr)->grank); } ret = 0; Modified: mvapich/branches/0.9.9/mpid/ch_gen2/viutil.h =================================================================== --- mvapich/branches/0.9.9/mpid/ch_gen2/viutil.h 2007-05-29 01:58:05 UTC (rev 1282) +++ mvapich/branches/0.9.9/mpid/ch_gen2/viutil.h 2007-05-29 07:47:10 UTC (rev 1283) @@ -48,7 +48,8 @@ } \ fprintf(stderr, message, ##args); \ fprintf(stderr, " at line %d in file %s\n", __LINE__, __FILE__);\ - pmgr_abort(); \ + sleep(1); \ + pmgr_abort(code); \ exit(code); \ }