From koop at mvapich.cse.ohio-state.edu Sun Dec 2 15:13:07 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Sun Dec 2 15:13:47 2007 Subject: [mvapich-commit] r1639 - mvapich/trunk/mpid/ch_gen2_ud Message-ID: <200712022013.lB2KD7DV002449@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-02 15:13:06 -0500 (Sun, 02 Dec 2007) New Revision: 1639 Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c Log: * Fixes race condition in deallocation of sbuf for UD-ZCopy finish message Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c =================================================================== --- mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c 2007-11-30 23:05:05 UTC (rev 1638) +++ mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c 2007-12-02 20:13:06 UTC (rev 1639) @@ -351,7 +351,11 @@ mv_sbuf * v = (mv_sbuf *) wc_list[i].wr_id; v->left_to_send--; if(0 == v->left_to_send) { - release_mv_sbuf(v); + v->in_progress = 0; + /* other side has sent ack back */ + if(0 == v->seqnum) { + release_mv_sbuf(v); + } } } } From koop at mvapich.cse.ohio-state.edu Mon Dec 3 20:53:00 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Mon Dec 3 20:53:41 2007 Subject: [mvapich-commit] r1640 - mvapich/trunk/mpid/ch_gen2 Message-ID: <200712040153.lB41r0TD005957@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-03 20:52:59 -0500 (Mon, 03 Dec 2007) New Revision: 1640 Modified: mvapich/trunk/mpid/ch_gen2/async_progress.c mvapich/trunk/mpid/ch_gen2/viacoalesce.c mvapich/trunk/mpid/ch_gen2/viaparam.c mvapich/trunk/mpid/ch_gen2/viaparam.h Log: * fix potential issue with coalescing and ASYNC mode * make default spin time for async QP configurable and set to a default of 32 (instead of 3000) Modified: mvapich/trunk/mpid/ch_gen2/async_progress.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-02 20:13:06 UTC (rev 1639) +++ mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-04 01:52:59 UTC (rev 1640) @@ -578,7 +578,7 @@ /* No completions! */ nspin++; - if(nspin == viadev_max_spin_count) { + if(nspin == viadev_max_async_spin_count) { break; } } Modified: mvapich/trunk/mpid/ch_gen2/viacoalesce.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viacoalesce.c 2007-12-02 20:13:06 UTC (rev 1639) +++ mvapich/trunk/mpid/ch_gen2/viacoalesce.c 2007-12-04 01:52:59 UTC (rev 1640) @@ -113,7 +113,11 @@ * packet */ h->pkt_count = 0; + + ASYNC_THREAD_LOCK; h->header.id = c->next_packet_tosend++; + ASYNC_THREAD_UNLOCK; + h->header.type = VIADEV_PACKET_EAGER_COALESCE; h->header.src_rank = viadev.me; Modified: mvapich/trunk/mpid/ch_gen2/viaparam.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-12-02 20:13:06 UTC (rev 1639) +++ mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-12-04 01:52:59 UTC (rev 1640) @@ -345,6 +345,8 @@ int viadev_use_blocking = 0; unsigned long long viadev_max_spin_count = VIADEV_MAX_SPIN_COUNT; +unsigned int viadev_max_async_spin_count = 32; + #ifdef _SMP_ int disable_shared_mem = 0; int smp_num_send_buffer = SMP_NUM_SEND_BUFFER; @@ -985,6 +987,10 @@ viadev_max_spin_count = atoll(value); } + if ((value = getenv("VIADEV_MAX_ASYNC_SPIN_COUNT")) != NULL) { + viadev_max_async_spin_count = atoi(value); + } + /* Default parameters in the presence of memory * based reliability */ @@ -1286,8 +1292,6 @@ error_abort_all(IBV_RETURN_ERR, "Could not autodetect HCA, try VIADEV_COMPAT_MODE=1\n"); break; - default: - viadev_set_compat_params(); } } else { Modified: mvapich/trunk/mpid/ch_gen2/viaparam.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/viaparam.h 2007-12-02 20:13:06 UTC (rev 1639) +++ mvapich/trunk/mpid/ch_gen2/viaparam.h 2007-12-04 01:52:59 UTC (rev 1640) @@ -151,6 +151,7 @@ extern int disable_lmc; extern unsigned long long viadev_max_spin_count; +extern unsigned int viadev_max_async_spin_count; extern int viadev_use_blocking; extern uint32_t viadev_debug_level; From koop at mvapich.cse.ohio-state.edu Wed Dec 5 17:05:19 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Wed Dec 5 17:06:00 2007 Subject: [mvapich-commit] r1641 - mvapich/trunk/mpid/ch_gen2 Message-ID: <200712052205.lB5M5J58011384@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-05 17:05:14 -0500 (Wed, 05 Dec 2007) New Revision: 1641 Modified: mvapich/trunk/mpid/ch_gen2/viaparam.c Log: * Turn off the memory registration cache if PTMALLOC is disabled Thanks to Adam Moody of LLNL for suggesting this change Modified: mvapich/trunk/mpid/ch_gen2/viaparam.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-12-04 01:52:59 UTC (rev 1640) +++ mvapich/trunk/mpid/ch_gen2/viaparam.c 2007-12-05 22:05:14 UTC (rev 1641) @@ -364,7 +364,11 @@ * * By default, it is turned on. Set to 0 to disable. */ +#ifndef DISABLE_PTMALLOC int viadev_use_dreg_cache = 1; +#else /* disable dreg cache if no PTMALLOC */ +int viadev_use_dreg_cache = 0; +#endif /* * Initialization parameters. @@ -638,7 +642,9 @@ if ((value = getenv("VIADEV_NDREG_ENTRIES")) != NULL) { viadev_ndreg_entries = (unsigned int)atoi(value); } - +#ifndef DISABLE_PTMALLOC + /* if PTMALLOC is turned off, we set viadev_use_dreg_cache=0 above + don't allow them to turn it back on here */ if ((value = getenv("VIADEV_USE_DREG_CACHE")) != NULL) { viadev_use_dreg_cache = (int)atoi(value); @@ -651,6 +657,7 @@ viadev_use_dreg_cache = 0; } } +#endif if ((value = getenv("VIADEV_SQ_SIZE")) != NULL) { viadev_sq_size = atol(value); } From chail at mvapich.cse.ohio-state.edu Wed Dec 5 17:29:59 2007 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Wed Dec 5 17:30:45 2007 Subject: [mvapich-commit] r1642 - mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2 Message-ID: <200712052229.lB5MTxNQ011438@mvapich.cse.ohio-state.edu> Author: chail Date: 2007-12-05 17:29:58 -0500 (Wed, 05 Dec 2007) New Revision: 1642 Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h Log: -- Explicitly name unions in struct ibv_wr_descriptor and reference the members in the code properly. -- Change "inline" functions to "static inline" properly. Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c 2007-12-05 22:29:58 UTC (rev 1642) @@ -124,7 +124,7 @@ return v; } -inline int PKT_IS_NOOP(void *v) +static inline int PKT_IS_NOOP(void *v) { MPIDI_CH3I_MRAILI_Pkt_comm_header * p = ((vbuf *)v)->pheader; return ((p->type == MPIDI_CH3_PKT_NOOP)? 1 : 0); Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-05 22:29:58 UTC (rev 1642) @@ -207,7 +207,7 @@ /* mark MPI send complete when VIA send completes */ - DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.sr.wr_id); + DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.u.sr.wr_id); } void MRAILI_RDMA_Get_finish(MPIDI_VC_t * vc, @@ -238,7 +238,7 @@ buf->sreq = (void *) rreq; - DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.sr.wr_id); + DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.u.sr.wr_id); } void MPIDI_CH3I_MRAILI_Rendezvous_rget_push(MPIDI_VC_t * vc, Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-05 22:29:58 UTC (rev 1642) @@ -33,7 +33,7 @@ #define DEBUG_PRINT(args...) #endif -inline int MRAILI_Coalesce_ok(MPIDI_VC_t * vc, int rail) { +static inline int MRAILI_Coalesce_ok(MPIDI_VC_t * vc, int rail) { if(rdma_use_coalesce && (vc->mrail.outstanding_eager_vbufs >= rdma_coalesce_threshold || @@ -466,7 +466,7 @@ rdma_num_ports * rdma_num_qp_per_port); if(ibv_post_srq_recv(MPIDI_CH3I_RDMA_Process.srq_hndl[hca_num], - &v->desc.rr, &bad_wr)) { + &v->desc.u.rr, &bad_wr)) { MRAILI_Release_vbuf(v); break; } Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c 2007-12-05 22:29:58 UTC (rev 1642) @@ -997,7 +997,7 @@ return ret; } -inline int round_left(int current, int size) +static inline int round_left(int current, int size) { if (current == 0) { return size-1; @@ -1006,7 +1006,7 @@ return current-1; } -inline int is_A_on_left_of_B(int a, int b, int rank, int size) +static inline int is_A_on_left_of_B(int a, int b, int rank, int size) { int dist_a = (rank - a + size)%size; int dist_b = (rank - b + size)%size; Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-05 22:29:58 UTC (rev 1642) @@ -207,12 +207,12 @@ { \ int __ret; \ if(((_v)->desc.sg_entry.length <= rdma_max_inline_size) \ - && ((_v)->desc.sr.opcode != IBV_WR_RDMA_READ)){ \ - (_v)->desc.sr.send_flags = (enum ibv_send_flags) \ + && ((_v)->desc.u.sr.opcode != IBV_WR_RDMA_READ)){ \ + (_v)->desc.u.sr.send_flags = (enum ibv_send_flags) \ (IBV_SEND_SIGNALED | \ IBV_SEND_INLINE); \ } else { \ - (_v)->desc.sr.send_flags = IBV_SEND_SIGNALED ; \ + (_v)->desc.u.sr.send_flags = IBV_SEND_SIGNALED ; \ } \ if ((_rail) != (_v)->rail) { \ DEBUG_PRINT(stderr, "[%s:%d] rail %d, vrail %d\n",\ @@ -220,7 +220,7 @@ assert((_rail) == (_v)->rail); \ } \ __ret = ibv_post_send((_c)->mrail.rails[(_rail)].qp_hndl,\ - &((_v)->desc.sr),&((_v)->desc.bad_sr)); \ + &((_v)->desc.u.sr),&((_v)->desc.y.bad_sr)); \ if(__ret) { \ fprintf(stderr, "failed while avail wqe is %d, " \ "rail %d\n", \ @@ -235,8 +235,8 @@ int __ret; \ _vbuf->vc = (void *)_c; \ __ret = ibv_post_recv(_c->mrail.rails[(_rail)].qp_hndl, \ - &((_vbuf)->desc.rr), \ - &((_vbuf)->desc.bad_rr)); \ + &((_vbuf)->desc.u.rr), \ + &((_vbuf)->desc.y.bad_rr)); \ if (__ret) { \ ibv_error_abort(IBV_RETURN_ERR, \ "ibv_post_recv err with %d", \ Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-05 22:29:58 UTC (rev 1642) @@ -371,13 +371,13 @@ void vbuf_init_rdma_write(vbuf * v) { - v->desc.sr.next = NULL; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->padding = FREE_FLAG; } @@ -385,12 +385,12 @@ { int hca_num = rail / (rdma_num_rails/rdma_num_hcas); - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_SEND; - v->desc.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_SEND; + v->desc.u.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = v->region->mem_handle[hca_num]->lkey; v->desc.sg_entry.addr = (uintptr_t)(v->buffer); @@ -401,10 +401,10 @@ { int hca_num = rail / (rdma_num_rails/rdma_num_hcas); - v->desc.rr.next = NULL; - v->desc.rr.wr_id = (uintptr_t) v; - v->desc.rr.num_sge = 1; - v->desc.rr.sg_list = &(v->desc.sg_entry); + v->desc.u.rr.next = NULL; + v->desc.u.rr.wr_id = (uintptr_t) v; + v->desc.u.rr.num_sge = 1; + v->desc.u.rr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = v->region->mem_handle[hca_num]->lkey; v->desc.sg_entry.addr = (uintptr_t)(v->buffer); @@ -417,17 +417,17 @@ void *remote_address, uint32_t rkey, int len, int rail) { - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_RDMA_READ; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_RDMA_READ; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.wr.rdma.remote_addr + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.wr.rdma.remote_addr = (uintptr_t)(remote_address); - v->desc.sr.wr.rdma.rkey = rkey; + v->desc.u.sr.wr.rdma.rkey = rkey; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = lkey; v->desc.sg_entry.addr = (uintptr_t)(local_address); @@ -442,17 +442,17 @@ void *remote_address, uint32_t rkey, int len, int rail) { - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.wr.rdma.remote_addr + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.wr.rdma.remote_addr = (uintptr_t)(remote_address); - v->desc.sr.wr.rdma.rkey = rkey; + v->desc.u.sr.wr.rdma.rkey = rkey; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = lkey; v->desc.sg_entry.addr = (uintptr_t)(local_address); Modified: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h 2007-12-05 22:05:14 UTC (rev 1641) +++ mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h 2007-12-05 22:29:58 UTC (rev 1642) @@ -79,11 +79,11 @@ union { struct ibv_recv_wr rr; struct ibv_send_wr sr; - }; + } u; union { struct ibv_send_wr * bad_sr; struct ibv_recv_wr * bad_rr; - }; + } y; struct ibv_sge sg_entry; void *next; }; @@ -152,16 +152,16 @@ void *remote_addr, uint32_t rkey) { - v->desc.sr.next = NULL; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); - (v)->desc.sr.wr.rdma.remote_addr = (uintptr_t) (remote_addr); - (v)->desc.sr.wr.rdma.rkey = (rkey); + (v)->desc.u.sr.wr.rdma.remote_addr = (uintptr_t) (remote_addr); + (v)->desc.u.sr.wr.rdma.rkey = (rkey); (v)->desc.sg_entry.length = (len); (v)->desc.sg_entry.lkey = (lkey); (v)->desc.sg_entry.addr = (uintptr_t)(local_addr); From koop at mvapich.cse.ohio-state.edu Wed Dec 5 18:08:12 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Wed Dec 5 18:08:53 2007 Subject: [mvapich-commit] r1643 - mvapich/trunk/mpid/ch_gen2_ud Message-ID: <200712052308.lB5N8C74011507@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-05 18:08:11 -0500 (Wed, 05 Dec 2007) New Revision: 1643 Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c Log: * wr_id is a mv_sdescriptor pointer -- not a mv_sbuf pointer Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c =================================================================== --- mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c 2007-12-05 22:29:58 UTC (rev 1642) +++ mvapich/trunk/mpid/ch_gen2_ud/mv_ud_zcopy.c 2007-12-05 23:08:11 UTC (rev 1643) @@ -348,7 +348,8 @@ qp->send_wqes_avail++; if(wc_list[i].wr_id) { - mv_sbuf * v = (mv_sbuf *) wc_list[i].wr_id; + mv_sbuf * v = (mv_sbuf *) + ((mv_sdescriptor *) wc_list[i].wr_id)->parent; v->left_to_send--; if(0 == v->left_to_send) { v->in_progress = 0; From chail at mvapich.cse.ohio-state.edu Wed Dec 5 21:04:19 2007 From: chail at mvapich.cse.ohio-state.edu (chail@mvapich.cse.ohio-state.edu) Date: Wed Dec 5 21:05:01 2007 Subject: [mvapich-commit] r1644 - mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2 Message-ID: <200712060204.lB624JvT011752@mvapich.cse.ohio-state.edu> Author: chail Date: 2007-12-05 21:04:17 -0500 (Wed, 05 Dec 2007) New Revision: 1644 Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h Log: -- Explicitly name unions in struct ibv_wr_descriptor and reference the members in the code properly. -- Change "inline" functions to "static inline" properly. Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_channel_manager.c 2007-12-06 02:04:17 UTC (rev 1644) @@ -124,7 +124,7 @@ return v; } -inline int PKT_IS_NOOP(void *v) +static inline int PKT_IS_NOOP(void *v) { MPIDI_CH3I_MRAILI_Pkt_comm_header * p = ((vbuf *)v)->pheader; return ((p->type == MPIDI_CH3_PKT_NOOP)? 1 : 0); Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-06 02:04:17 UTC (rev 1644) @@ -207,7 +207,7 @@ /* mark MPI send complete when VIA send completes */ - DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.sr.wr_id); + DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.u.sr.wr_id); } void MRAILI_RDMA_Get_finish(MPIDI_VC_t * vc, @@ -238,7 +238,7 @@ buf->sreq = (void *) rreq; - DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.sr.wr_id); + DEBUG_PRINT("VBUF ASSOCIATED: %p, %08x\n", buf, buf->desc.u.sr.wr_id); } void MPIDI_CH3I_MRAILI_Rendezvous_rget_push(MPIDI_VC_t * vc, Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-06 02:04:17 UTC (rev 1644) @@ -33,7 +33,7 @@ #define DEBUG_PRINT(args...) #endif -inline int MRAILI_Coalesce_ok(MPIDI_VC_t * vc, int rail) { +static inline int MRAILI_Coalesce_ok(MPIDI_VC_t * vc, int rail) { if(rdma_use_coalesce && (vc->mrail.outstanding_eager_vbufs >= rdma_coalesce_threshold || @@ -466,7 +466,7 @@ rdma_num_ports * rdma_num_qp_per_port); if(ibv_post_srq_recv(MPIDI_CH3I_RDMA_Process.srq_hndl[hca_num], - &v->desc.rr, &bad_wr)) { + &v->desc.u.rr, &bad_wr)) { MRAILI_Release_vbuf(v); break; } Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_iba_priv.c 2007-12-06 02:04:17 UTC (rev 1644) @@ -997,7 +997,7 @@ return ret; } -inline int round_left(int current, int size) +static inline int round_left(int current, int size) { if (current == 0) { return size-1; @@ -1006,7 +1006,7 @@ return current-1; } -inline int is_A_on_left_of_B(int a, int b, int rank, int size) +static inline int is_A_on_left_of_B(int a, int b, int rank, int size) { int dist_a = (rank - a + size)%size; int dist_b = (rank - b + size)%size; Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-06 02:04:17 UTC (rev 1644) @@ -207,12 +207,12 @@ { \ int __ret; \ if(((_v)->desc.sg_entry.length <= rdma_max_inline_size) \ - && ((_v)->desc.sr.opcode != IBV_WR_RDMA_READ)){ \ - (_v)->desc.sr.send_flags = (enum ibv_send_flags) \ + && ((_v)->desc.u.sr.opcode != IBV_WR_RDMA_READ)){ \ + (_v)->desc.u.sr.send_flags = (enum ibv_send_flags) \ (IBV_SEND_SIGNALED | \ IBV_SEND_INLINE); \ } else { \ - (_v)->desc.sr.send_flags = IBV_SEND_SIGNALED ; \ + (_v)->desc.u.sr.send_flags = IBV_SEND_SIGNALED ; \ } \ if ((_rail) != (_v)->rail) { \ DEBUG_PRINT(stderr, "[%s:%d] rail %d, vrail %d\n",\ @@ -220,7 +220,7 @@ assert((_rail) == (_v)->rail); \ } \ __ret = ibv_post_send((_c)->mrail.rails[(_rail)].qp_hndl,\ - &((_v)->desc.sr),&((_v)->desc.bad_sr)); \ + &((_v)->desc.u.sr),&((_v)->desc.y.bad_sr)); \ if(__ret) { \ fprintf(stderr, "failed while avail wqe is %d, " \ "rail %d\n", \ @@ -235,8 +235,8 @@ int __ret; \ _vbuf->vc = (void *)_c; \ __ret = ibv_post_recv(_c->mrail.rails[(_rail)].qp_hndl, \ - &((_vbuf)->desc.rr), \ - &((_vbuf)->desc.bad_rr)); \ + &((_vbuf)->desc.u.rr), \ + &((_vbuf)->desc.y.bad_rr)); \ if (__ret) { \ ibv_error_abort(IBV_RETURN_ERR, \ "ibv_post_recv err with %d", \ Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-06 02:04:17 UTC (rev 1644) @@ -369,13 +369,13 @@ void vbuf_init_rdma_write(vbuf * v) { - v->desc.sr.next = NULL; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->padding = FREE_FLAG; } @@ -383,12 +383,12 @@ { int hca_num = rail / (rdma_num_rails/rdma_num_hcas); - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_SEND; - v->desc.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_SEND; + v->desc.u.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = v->region->mem_handle[hca_num]->lkey; v->desc.sg_entry.addr = (uintptr_t)(v->buffer); @@ -399,10 +399,10 @@ { int hca_num = rail / (rdma_num_rails/rdma_num_hcas); - v->desc.rr.next = NULL; - v->desc.rr.wr_id = (uintptr_t) v; - v->desc.rr.num_sge = 1; - v->desc.rr.sg_list = &(v->desc.sg_entry); + v->desc.u.rr.next = NULL; + v->desc.u.rr.wr_id = (uintptr_t) v; + v->desc.u.rr.num_sge = 1; + v->desc.u.rr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = v->region->mem_handle[hca_num]->lkey; v->desc.sg_entry.addr = (uintptr_t)(v->buffer); @@ -415,17 +415,17 @@ void *remote_address, uint32_t rkey, int len, int rail) { - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_RDMA_READ; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_RDMA_READ; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.wr.rdma.remote_addr + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.wr.rdma.remote_addr = (uintptr_t)(remote_address); - v->desc.sr.wr.rdma.rkey = rkey; + v->desc.u.sr.wr.rdma.rkey = rkey; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = lkey; v->desc.sg_entry.addr = (uintptr_t)(local_address); @@ -440,17 +440,17 @@ void *remote_address, uint32_t rkey, int len, int rail) { - v->desc.sr.next = NULL; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.wr.rdma.remote_addr + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.wr.rdma.remote_addr = (uintptr_t)(remote_address); - v->desc.sr.wr.rdma.rkey = rkey; + v->desc.u.sr.wr.rdma.rkey = rkey; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.sg_list = &(v->desc.sg_entry); v->desc.sg_entry.length = len; v->desc.sg_entry.lkey = lkey; v->desc.sg_entry.addr = (uintptr_t)(local_address); Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h 2007-12-05 23:08:11 UTC (rev 1643) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.h 2007-12-06 02:04:17 UTC (rev 1644) @@ -79,11 +79,11 @@ union { struct ibv_recv_wr rr; struct ibv_send_wr sr; - }; + } u; union { struct ibv_send_wr * bad_sr; struct ibv_recv_wr * bad_rr; - }; + } y; struct ibv_sge sg_entry; void *next; }; @@ -152,16 +152,16 @@ void *remote_addr, uint32_t rkey) { - v->desc.sr.next = NULL; - v->desc.sr.opcode = IBV_WR_RDMA_WRITE; - v->desc.sr.send_flags = IBV_SEND_SIGNALED; - v->desc.sr.wr_id = (uintptr_t) v; + v->desc.u.sr.next = NULL; + v->desc.u.sr.opcode = IBV_WR_RDMA_WRITE; + v->desc.u.sr.send_flags = IBV_SEND_SIGNALED; + v->desc.u.sr.wr_id = (uintptr_t) v; - v->desc.sr.num_sge = 1; - v->desc.sr.sg_list = &(v->desc.sg_entry); + v->desc.u.sr.num_sge = 1; + v->desc.u.sr.sg_list = &(v->desc.sg_entry); - (v)->desc.sr.wr.rdma.remote_addr = (uintptr_t) (remote_addr); - (v)->desc.sr.wr.rdma.rkey = (rkey); + (v)->desc.u.sr.wr.rdma.remote_addr = (uintptr_t) (remote_addr); + (v)->desc.u.sr.wr.rdma.rkey = (rkey); (v)->desc.sg_entry.length = (len); (v)->desc.sg_entry.lkey = (lkey); (v)->desc.sg_entry.addr = (uintptr_t)(local_addr); From koop at mvapich.cse.ohio-state.edu Thu Dec 6 12:03:46 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Thu Dec 6 12:04:27 2007 Subject: [mvapich-commit] r1645 - in mvapich/trunk/mpid/ch_gen2: . process Message-ID: <200712061703.lB6H3kkC013902@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-06 12:03:44 -0500 (Thu, 06 Dec 2007) New Revision: 1645 Modified: mvapich/trunk/mpid/ch_gen2/cm_user.c mvapich/trunk/mpid/ch_gen2/cm_user.h mvapich/trunk/mpid/ch_gen2/process/minidaemon.h mvapich/trunk/mpid/ch_gen2/process/minidaemon_client.c mvapich/trunk/mpid/ch_gen2/viacoalesce.c mvapich/trunk/mpid/ch_gen2/viapriv.h Log: * Changes to allow Sun Studio compiler support Modified: mvapich/trunk/mpid/ch_gen2/cm_user.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/cm_user.c 2007-12-06 02:04:17 UTC (rev 1644) +++ mvapich/trunk/mpid/ch_gen2/cm_user.c 2007-12-06 17:03:44 UTC (rev 1645) @@ -24,26 +24,6 @@ #include "cm.h" #include "cm_user.h" -inline void odu_test_new_connection() -{ - int i; - if (viadev.cm_new_connection == 0) - return; - /* Note: to avoid race condition, must first put down flag, then process - * pending sends*/ - viadev.cm_new_connection = 0; - - for (i=0; i -inline void odu_test_new_connection(void); - void odu_enable_qp(int peer, struct ibv_qp * qp); void cm_process_queue(int peer_rank); +static inline void odu_test_new_connection() +{ + int i; + if (viadev.cm_new_connection == 0) + return; + /* Note: to avoid race condition, must first put down flag, then process + * pending sends*/ + viadev.cm_new_connection = 0; + + for (i=0; i*/ -#include "mpirun_rsh.h" -#define MD_USR_ERROR(msg) \ - do { \ - fprintf(stderr,"%s\n",msg); \ - exit(1); \ - } while (0); - -#define MD_SYS_ERROR(msg) \ - do { \ - perror(msg); \ - exit(1); \ - } while (0); - -/* Support for debug prints. */ - -/*#define DNONE 0*/ /* Message with this debug level will be always printed */ -/*#define DINFO 1*/ /* Info messages level is applicable for rare and informative messages */ -/*#define DDEBUG 2 *//* Debug level, should be turned off in normal run */ -/*#define DPATH 3 *//* The highest level, for use in loops or critical sections */ - -/*#define DGLOBAL_LEVEL 3*/ - -/* DGLOBAL_LEVEL should be defined in compilation stage, otherwise it should be zero */ -/*#define MD_PRINT(dlevel,fmt, args...) {if (dlevel <= DGLOBAL_LEVEL){\ - fprintf(stderr, "[%s:%d, pid=%d]", __FILE__, __LINE__,getpid());\ - fprintf(stderr, fmt, ## args); fflush(stderr);}}*/ - - - - -/** -Minidaemon ADT provided by Mellanox, MPI Team - -Minidaemon interface provides the possibilty to create and run -daemon process on clusters node. Each instance of it will be created -at application run time and will destroyed right after application termination. -**/ -/*typedef enum { - P_NOTSTARTED, - P_STARTED, - P_CONNECTED, - P_DISCONNECTED, - P_RUNNING, - P_FINISHED, - P_EXITED -} process_state; - -struct process_t { - char *hostname; - char *device; - pid_t pid; - pid_t remote_pid; - int port; - int control_socket; - process_state state; -} ;*/ - -/* Minidaemon Data Structures */ - -typedef struct childrenList_t * ChildrenList; -typedef struct md_entry_t * MD_entry; -typedef struct minidaemon_t * Minidaemon; -/*typedef struct process_t process;*/ - -void minidaemon_create(process * procList, int nproc, int width,const char * command_params,const char * command, int mpirun_port,int ppid); - -/* Init all data member structures */ -void minidaemon_init (const char * par_name , int ch_num, int width, const char * command_params,const char * command, - const char * root_hostname, int root_ch_num,int mpirun_port, int ppid, int array_it); - -/* Start listening to messages from other Minidaemons */ -void minidaemon_run (); - -#endif /* _MINIDAEMON_H */ +#ifndef _MINIDAEMON_H +#define _MINIDAEMON_H 1 + +/** +* Minidaemon ADT provided by Mellanox, MPI Team +* mailto : xalex@mellanox.co.il +* +* Minidaemon interface provides the possibilty to create and run +* daemon process on clusters node. Each instance of it will be created +* at application run time and will destroyed right after application termination. +* Each minidaemon instance may run user-defined jobs on a local machine as well as launch other minidaemons on remote machines. +* So, the user can define the execution scheme ("launch tree") for his needs. The concrete +* Thus, Minidaemon provide an answer to the following issues : + 1. Multicore management. It's only one daemon per machine that will start all jobs there + 2. Scalable and fast launch of (MPI) jobs on a cluster, using different start-up schemes (flat, multilevel tree) + 3. Cluster job management and cleanup (this version is ad-hoc MPI tuned) + 4. Ease of use and simplicity for cluster administrators. There's no constant daemon that requires + additional management and resources +**/ + +/*#include */ +#include "mpirun_rsh.h" +#define MD_USR_ERROR(msg) \ + do { \ + fprintf(stderr,"%s\n",msg); \ + exit(1); \ + } while (0); + +#define MD_SYS_ERROR(msg) \ + do { \ + perror(msg); \ + exit(1); \ + } while (0); + +/* Support for debug prints. */ + +/*#define DNONE 0*/ /* Message with this debug level will be always printed */ +/*#define DINFO 1*/ /* Info messages level is applicable for rare and informative messages */ +/*#define DDEBUG 2 *//* Debug level, should be turned off in normal run */ +/*#define DPATH 3 *//* The highest level, for use in loops or critical sections */ + +/*#define DGLOBAL_LEVEL 3*/ + +/* DGLOBAL_LEVEL should be defined in compilation stage, otherwise it should be zero */ +/*#define MD_PRINT(dlevel,fmt, args...) {if (dlevel <= DGLOBAL_LEVEL){\ + fprintf(stderr, "[%s:%d, pid=%d]", __FILE__, __LINE__,getpid());\ + fprintf(stderr, fmt, ## args); fflush(stderr);}}*/ + + + + +/** +Minidaemon ADT provided by Mellanox, MPI Team + +Minidaemon interface provides the possibilty to create and run +daemon process on clusters node. Each instance of it will be created +at application run time and will destroyed right after application termination. +**/ +/*typedef enum { + P_NOTSTARTED, + P_STARTED, + P_CONNECTED, + P_DISCONNECTED, + P_RUNNING, + P_FINISHED, + P_EXITED +} process_state; + +struct process_t { + char *hostname; + char *device; + pid_t pid; + pid_t remote_pid; + int port; + int control_socket; + process_state state; +} ;*/ + +/* Minidaemon Data Structures */ + +typedef struct childrenList_t * ChildrenList; +typedef struct md_entry_t * MD_entry; +typedef struct minidaemon_t * Minidaemon; +/*typedef struct process_t process;*/ + +void minidaemon_create(process * procList, int nproc, int width,const char * command_params,const char * command, int mpirun_port,int ppid); + +/* Init all data member structures */ +void minidaemon_init (const char * par_name , int ch_num, int width, const char * command_params,const char * command, + const char * root_hostname, int root_ch_num,int mpirun_port, int ppid, int array_it); + +/* Start listening to messages from other Minidaemons */ +void minidaemon_run (); + +#endif /* _MINIDAEMON_H */ Modified: mvapich/trunk/mpid/ch_gen2/process/minidaemon_client.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/minidaemon_client.c 2007-12-06 02:04:17 UTC (rev 1644) +++ mvapich/trunk/mpid/ch_gen2/process/minidaemon_client.c 2007-12-06 17:03:44 UTC (rev 1645) @@ -1,31 +1,31 @@ -#include -#include "minidaemon.h" -#define DEFAULT_PARAM_NUMBER 10 - -#define NAME_INDEX 1 -#define NUM_INDEX 2 -#define WIDTH_INDEX 3 -#define COM_PAR_INDEX 4 -#define COM_INDEX 5 -#define ROOT_NAME 6 -#define ROOT_CH_NUM 7 -#define MPIRUN_PORT_INDEX 8 -#define PID_INDEX 9 -#define ARRAY_IT_INDEX 10 - -int main (int argc, char * argv[]) { - int width; - if (argc < DEFAULT_PARAM_NUMBER) { - MD_USR_ERROR("Minidaemon client : too few parameters to run minidaemon"); - } - if ( (width = atoi(argv[WIDTH_INDEX])) < 1) { - MD_USR_ERROR("Minidaemon client : invalid tree width received"); - } - /** Minidaemon functions will exit on error **/ - - /*void minidaemon_init (const char * par_name , int ch_num, int width, const char * command_params,const char * command, int root_ch_num);*/ - minidaemon_init(argv[NAME_INDEX],atoi(argv[NUM_INDEX]),atoi(argv[WIDTH_INDEX]), argv[COM_PAR_INDEX],argv[COM_INDEX], - argv[ROOT_NAME],atoi(argv[ROOT_CH_NUM]),atoi(argv[MPIRUN_PORT_INDEX]), atoi(argv[PID_INDEX]), atoi(argv[ARRAY_IT_INDEX])); - minidaemon_run(); - return 0; -} +#include +#include "minidaemon.h" +#define DEFAULT_PARAM_NUMBER 10 + +#define NAME_INDEX 1 +#define NUM_INDEX 2 +#define WIDTH_INDEX 3 +#define COM_PAR_INDEX 4 +#define COM_INDEX 5 +#define ROOT_NAME 6 +#define ROOT_CH_NUM 7 +#define MPIRUN_PORT_INDEX 8 +#define PID_INDEX 9 +#define ARRAY_IT_INDEX 10 + +int main (int argc, char * argv[]) { + int width; + if (argc < DEFAULT_PARAM_NUMBER) { + MD_USR_ERROR("Minidaemon client : too few parameters to run minidaemon"); + } + if ( (width = atoi(argv[WIDTH_INDEX])) < 1) { + MD_USR_ERROR("Minidaemon client : invalid tree width received"); + } + /** Minidaemon functions will exit on error **/ + + /*void minidaemon_init (const char * par_name , int ch_num, int width, const char * command_params,const char * command, int root_ch_num);*/ + minidaemon_init(argv[NAME_INDEX],atoi(argv[NUM_INDEX]),atoi(argv[WIDTH_INDEX]), argv[COM_PAR_INDEX],argv[COM_INDEX], + argv[ROOT_NAME],atoi(argv[ROOT_CH_NUM]),atoi(argv[MPIRUN_PORT_INDEX]), atoi(argv[PID_INDEX]), atoi(argv[ARRAY_IT_INDEX])); + minidaemon_run(); + return 0; +} Modified: mvapich/trunk/mpid/ch_gen2/viacoalesce.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/viacoalesce.c 2007-12-06 02:04:17 UTC (rev 1644) +++ mvapich/trunk/mpid/ch_gen2/viacoalesce.c 2007-12-06 17:03:44 UTC (rev 1645) @@ -136,21 +136,7 @@ return to_use; } -inline int eager_coalesce_ok(viadev_connection_t * c, int len) { - D_PRINT("threshold: %d, sq_size: %d, send_wqe: %d\n", - viadev_coalesce_threshold, viadev_sq_size, c->send_wqes_avail); - - if(viadev_use_eager_coalesce && - (viadev_sq_size - c->send_wqes_avail) >= viadev_coalesce_threshold && - len <= viadev_coalesce_threshold_size && - len <= (VBUF_DATA_SIZE(viadev_packet_eager_coalesce) - - sizeof(viadev_packet_eager_coalesce_full))) { - return 1; - } - return 0; -} - void eager_coalesce(viadev_connection_t * c, char * buf, int len, viadev_packet_envelope * envelope) { vbuf * v = NULL; Modified: mvapich/trunk/mpid/ch_gen2/viapriv.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/viapriv.h 2007-12-06 02:04:17 UTC (rev 1644) +++ mvapich/trunk/mpid/ch_gen2/viapriv.h 2007-12-06 17:03:44 UTC (rev 1645) @@ -584,7 +584,7 @@ void viadev_eager_pull(MPIR_RHANDLE * rhandle); void viadev_eager_coalesce_pull(MPIR_RHANDLE * rhandle); -inline int eager_coalesce_ok(viadev_connection_t * c, int len); + void eager_coalesce(viadev_connection_t * c, char * buf, int len, viadev_packet_envelope * envelope); void prepare_coalesced_pkt(viadev_connection_t * c, vbuf *v); @@ -649,6 +649,21 @@ int viadev_init_async_progress(void); int viadev_finalize_async_progress(void); +static inline int eager_coalesce_ok(viadev_connection_t * c, int len) { + + D_PRINT("threshold: %d, sq_size: %d, send_wqe: %d\n", + viadev_coalesce_threshold, viadev_sq_size, c->send_wqes_avail); + + if(viadev_use_eager_coalesce && + (viadev_sq_size - c->send_wqes_avail) >= viadev_coalesce_threshold && + len <= viadev_coalesce_threshold_size && + len <= (VBUF_BUFFER_SIZE - sizeof(viadev_packet_eager_coalesce) - + sizeof(viadev_packet_eager_coalesce_full))) { + return 1; + } + return 0; +} + /* * on 32 bit machines, we identify (for now) a handle by its address, using the * address for immediate data. From huangwei at mvapich.cse.ohio-state.edu Thu Dec 6 14:29:52 2007 From: huangwei at mvapich.cse.ohio-state.edu (huangwei@mvapich.cse.ohio-state.edu) Date: Thu Dec 6 14:30:35 2007 Subject: [mvapich-commit] r1646 - in mvapich2/branches/1.0: . src/mpid/osu_ch3/channels/mrail/src/gen2 Message-ID: <200712061929.lB6JTqQN014165@mvapich.cse.ohio-state.edu> Author: huangwei Date: 2007-12-06 14:29:50 -0500 (Thu, 06 Dec 2007) New Revision: 1646 Modified: mvapich2/branches/1.0/make.mvapich2.ofa mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c Log: Check in Matt's fix to branch Modified: mvapich2/branches/1.0/make.mvapich2.ofa =================================================================== --- mvapich2/branches/1.0/make.mvapich2.ofa 2007-12-06 17:03:44 UTC (rev 1645) +++ mvapich2/branches/1.0/make.mvapich2.ofa 2007-12-06 19:29:50 UTC (rev 1646) @@ -54,7 +54,7 @@ if [ ! -z "$BLCR_HOME" ]; then BLCR_LIB="-L$BLCR_HOME/lib -lcr" BLCR_INC="-I$BLCR_HOME/include" - CR_FLAG="-DCKPT" + CR_FLAG="-DCKPT -DSYNC_CKPT" SMP_FLAG="" export MPIEXEC_CR=mpiexec_cr # to compile mpiexec_cr LD_LIBRARY_PATH=$BLCR_HOME/lib:$LD_LIBRARY_PATH Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-06 17:03:44 UTC (rev 1645) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-06 19:29:50 UTC (rev 1646) @@ -129,8 +129,8 @@ if(NULL != (_vc)->mrail.coalesce_vbuf && \ (_vc)->mrail.coalesce_vbuf->rail == _rail) { \ MRAILI_Ext_sendq_send(_vc, (_vc)->mrail.coalesce_vbuf->rail); \ + (_vc)->mrail.coalesce_vbuf = NULL; \ } \ - (_vc)->mrail.coalesce_vbuf = NULL; \ } @@ -632,8 +632,8 @@ DEBUG_PRINT("returning back a coalesce buffer\n"); return vc->mrail.coalesce_vbuf; } else { + FLUSH_SQUEUE(vc); vc->mrail.coalesce_vbuf = NULL; - FLUSH_SQUEUE(vc); DEBUG_PRINT("Send out the coalesce vbuf\n"); } } Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-06 17:03:44 UTC (rev 1645) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-06 19:29:50 UTC (rev 1646) @@ -280,7 +280,10 @@ */ v->sreq = NULL; v->coalesce = 0; + v->content_size = 0; + v->eager = 0; + if (MPIDI_CH3I_RDMA_Process.has_srq #ifdef CKPT || 1 From koop at mvapich.cse.ohio-state.edu Thu Dec 6 14:46:27 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Thu Dec 6 14:47:08 2007 Subject: [mvapich-commit] r1647 - mvapich2/branches/1.0/src/include Message-ID: <200712061946.lB6JkRh7014195@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-06 14:46:25 -0500 (Thu, 06 Dec 2007) New Revision: 1647 Modified: mvapich2/branches/1.0/src/include/mpihandlemem.h Log: * Increase the maximum number of buffer allocations for communication intensive applications Modified: mvapich2/branches/1.0/src/include/mpihandlemem.h =================================================================== --- mvapich2/branches/1.0/src/include/mpihandlemem.h 2007-12-06 19:29:50 UTC (rev 1646) +++ mvapich2/branches/1.0/src/include/mpihandlemem.h 2007-12-06 19:46:25 UTC (rev 1647) @@ -94,7 +94,7 @@ /* Handle block is between 1 and 1024 *elements* */ #define HANDLE_BLOCK_SIZE 256 /* Index size is bewtween 1 and 65536 *elements* */ -#define HANDLE_BLOCK_INDEX_SIZE 1024 +#define HANDLE_BLOCK_INDEX_SIZE 4096 /* For direct, the remainder of the handle is the index into a predefined block */ From koop at mvapich.cse.ohio-state.edu Thu Dec 6 14:46:53 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Thu Dec 6 14:47:35 2007 Subject: [mvapich-commit] r1648 - mvapich2/trunk/src/include Message-ID: <200712061946.lB6JkrgY014207@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-06 14:46:53 -0500 (Thu, 06 Dec 2007) New Revision: 1648 Modified: mvapich2/trunk/src/include/mpihandlemem.h Log: * Increase the maximum number of buffer allocations for communication intensive applications Modified: mvapich2/trunk/src/include/mpihandlemem.h =================================================================== --- mvapich2/trunk/src/include/mpihandlemem.h 2007-12-06 19:46:25 UTC (rev 1647) +++ mvapich2/trunk/src/include/mpihandlemem.h 2007-12-06 19:46:53 UTC (rev 1648) @@ -94,7 +94,7 @@ /* Handle block is between 1 and 1024 *elements* */ #define HANDLE_BLOCK_SIZE 256 /* Index size is bewtween 1 and 65536 *elements* */ -#define HANDLE_BLOCK_INDEX_SIZE 1024 +#define HANDLE_BLOCK_INDEX_SIZE 4096 /* For direct, the remainder of the handle is the index into a predefined block */ From koop at mvapich.cse.ohio-state.edu Fri Dec 7 10:12:37 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Fri Dec 7 10:13:19 2007 Subject: [mvapich-commit] r1649 - mvapich/trunk/mpid/ch_gen2 Message-ID: <200712071512.lB7FCbpR016763@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-07 10:12:36 -0500 (Fri, 07 Dec 2007) New Revision: 1649 Modified: mvapich/trunk/mpid/ch_gen2/async_progress.c Log: * Request events and then poll until the CQ is empty -- not the other way around. Modified: mvapich/trunk/mpid/ch_gen2/async_progress.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-06 19:46:53 UTC (rev 1648) +++ mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-07 15:12:36 UTC (rev 1649) @@ -502,6 +502,27 @@ } +#define PROCESS_ASYNC_CQ_EVENT { \ + vbuf_addr = (void *) ((aint_t) sc.wr_id); \ + if (sc.status != IBV_WC_SUCCESS) { \ + error_abort_all(((vbuf *) vbuf_addr)->grank, \ + "[%s:%d] Got completion with error %s, " \ + "code=%d, dest rank=%d\n", \ + viadev.my_name, viadev.me, \ + wc_code_to_str(sc.status), sc.status, \ + ((vbuf *) vbuf_addr)->grank); \ + } \ + if (sc.opcode == IBV_WC_SEND || \ + sc.opcode == IBV_WC_RDMA_WRITE || \ + sc.opcode == IBV_WC_RDMA_READ) { \ + /* Send Completion */ \ + process_async_send_completion(&sc); \ + } else { \ + /* Recv Completion */ \ + process_async_recv_completion(&sc); \ + } \ +} + static void async_completion_thread(void * ctx) { int ret; @@ -537,45 +558,27 @@ ibv_ack_cq_events(viadev.async_info.cq, 1); + if (ibv_req_notify_cq(viadev.async_info.cq, 0)) { + error_abort_all(IBV_RETURN_ERR, + "Couldn't request for CQ notification\n"); + } + #if 0 fprintf(stderr,"[%d] Woke up!\n", viadev.me); #endif while (1) { - ne = ibv_poll_cq(viadev.async_info.cq, 1, &sc); if (ne > 0) { - vbuf_addr = (void *) ((aint_t) sc.wr_id); + PROCESS_ASYNC_CQ_EVENT; - if (sc.status != IBV_WC_SUCCESS) { - error_abort_all(((vbuf *) vbuf_addr)->grank, - "[%s:%d] Got completion with error %s, " - "code=%d, dest rank=%d\n", - viadev.my_name, viadev.me, - wc_code_to_str(sc.status), sc.status, - ((vbuf *) vbuf_addr)->grank); - } - /* Process the completion */ - - if (sc.opcode == IBV_WC_SEND || - sc.opcode == IBV_WC_RDMA_WRITE || - sc.opcode == IBV_WC_RDMA_READ) { - /* Send Completion */ - process_async_send_completion(&sc); - } else { - /* Recv Completion */ - process_async_recv_completion(&sc); - } - /* Found something, erase poll counter */ nspin = 0; } else if (ne < 0) { - error_abort_all(IBV_RETURN_ERR, - "[%s:%d] Got error polling CQ\n", + error_abort_all(IBV_RETURN_ERR, "[%s:%d] Got error polling CQ\n", viadev.my_name, viadev.me); } else { - /* No completions! */ nspin++; if(nspin == viadev_max_async_spin_count) { @@ -583,11 +586,6 @@ } } } - - if (ibv_req_notify_cq(viadev.async_info.cq, 0)) { - error_abort_all(IBV_RETURN_ERR, - "Couldn't request for CQ notification\n"); - } } } From koop at cse.ohio-state.edu Fri Dec 7 10:14:29 2007 From: koop at cse.ohio-state.edu (Matthew Koop) Date: Fri Dec 7 10:15:10 2007 Subject: [mvapich-commit] r1649 - mvapich/trunk/mpid/ch_gen2 In-Reply-To: <200712071512.lB7FCbpR016763@mvapich.cse.ohio-state.edu> Message-ID: I've run mpitest for 250 iterations overnight and mpitest didn't fail after this check-in. Jonathan, can you restart all ASYNC tests? Matt On Fri, 7 Dec 2007 koop@mvapich.cse.ohio-state.edu wrote: > Author: koop > Date: 2007-12-07 10:12:36 -0500 (Fri, 07 Dec 2007) > New Revision: 1649 > > Modified: > mvapich/trunk/mpid/ch_gen2/async_progress.c > Log: > > * Request events and then poll until the CQ is empty -- not the other > way around. > > > > Modified: mvapich/trunk/mpid/ch_gen2/async_progress.c > =================================================================== > --- mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-06 19:46:53 UTC (rev 1648) > +++ mvapich/trunk/mpid/ch_gen2/async_progress.c 2007-12-07 15:12:36 UTC (rev 1649) > @@ -502,6 +502,27 @@ > > } > > +#define PROCESS_ASYNC_CQ_EVENT { \ > + vbuf_addr = (void *) ((aint_t) sc.wr_id); \ > + if (sc.status != IBV_WC_SUCCESS) { \ > + error_abort_all(((vbuf *) vbuf_addr)->grank, \ > + "[%s:%d] Got completion with error %s, " \ > + "code=%d, dest rank=%d\n", \ > + viadev.my_name, viadev.me, \ > + wc_code_to_str(sc.status), sc.status, \ > + ((vbuf *) vbuf_addr)->grank); \ > + } \ > + if (sc.opcode == IBV_WC_SEND || \ > + sc.opcode == IBV_WC_RDMA_WRITE || \ > + sc.opcode == IBV_WC_RDMA_READ) { \ > + /* Send Completion */ \ > + process_async_send_completion(&sc); \ > + } else { \ > + /* Recv Completion */ \ > + process_async_recv_completion(&sc); \ > + } \ > +} > + > static void async_completion_thread(void * ctx) > { > int ret; > @@ -537,45 +558,27 @@ > > ibv_ack_cq_events(viadev.async_info.cq, 1); > > + if (ibv_req_notify_cq(viadev.async_info.cq, 0)) { > + error_abort_all(IBV_RETURN_ERR, > + "Couldn't request for CQ notification\n"); > + } > + > #if 0 > fprintf(stderr,"[%d] Woke up!\n", viadev.me); > #endif > > while (1) { > - > ne = ibv_poll_cq(viadev.async_info.cq, 1, &sc); > > if (ne > 0) { > - vbuf_addr = (void *) ((aint_t) sc.wr_id); > + PROCESS_ASYNC_CQ_EVENT; > > - if (sc.status != IBV_WC_SUCCESS) { > - error_abort_all(((vbuf *) vbuf_addr)->grank, > - "[%s:%d] Got completion with error %s, " > - "code=%d, dest rank=%d\n", > - viadev.my_name, viadev.me, > - wc_code_to_str(sc.status), sc.status, > - ((vbuf *) vbuf_addr)->grank); > - } > - /* Process the completion */ > - > - if (sc.opcode == IBV_WC_SEND || > - sc.opcode == IBV_WC_RDMA_WRITE || > - sc.opcode == IBV_WC_RDMA_READ) { > - /* Send Completion */ > - process_async_send_completion(&sc); > - } else { > - /* Recv Completion */ > - process_async_recv_completion(&sc); > - } > - > /* Found something, erase poll counter */ > nspin = 0; > } else if (ne < 0) { > - error_abort_all(IBV_RETURN_ERR, > - "[%s:%d] Got error polling CQ\n", > + error_abort_all(IBV_RETURN_ERR, "[%s:%d] Got error polling CQ\n", > viadev.my_name, viadev.me); > } else { > - /* No completions! */ > nspin++; > > if(nspin == viadev_max_async_spin_count) { > @@ -583,11 +586,6 @@ > } > } > } > - > - if (ibv_req_notify_cq(viadev.async_info.cq, 0)) { > - error_abort_all(IBV_RETURN_ERR, > - "Couldn't request for CQ notification\n"); > - } > } > } > > > _______________________________________________ > mvapich-commit mailing list > mvapich-commit@cse.ohio-state.edu > http://mail.cse.ohio-state.edu/mailman/listinfo/mvapich-commit > From curtisbr at mvapich.cse.ohio-state.edu Fri Dec 7 19:42:10 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Fri Dec 7 19:42:50 2007 Subject: [mvapich-commit] r1650 - in mvapich/trunk/mpid: ch_gen2/process util Message-ID: <200712080042.lB80gA18017567@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-07 19:42:08 -0500 (Fri, 07 Dec 2007) New Revision: 1650 Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c mvapich/trunk/mpid/util/tr2.h Log: Corrections to eliminate compilation warnings with Sun Studio 12 compiler. Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2007-12-07 15:12:36 UTC (rev 1649) +++ mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2007-12-08 00:42:08 UTC (rev 1650) @@ -1735,7 +1735,7 @@ for(i = 0; i < pglist->npgs; i++) { if(!(pglist->index[i]->pid = fork())) { - char *argv[5]; + const char* argv[5]; char *command; tmp = mkstr("%s MPISPAWN_ID=%d", mpispawn_env, i); @@ -1838,7 +1838,7 @@ argv[4] = NULL; } - execv(argv[0], argv); + execv(argv[0], (char* const*) argv); perror("execv"); for(i = 0; i < argc; i++) { Modified: mvapich/trunk/mpid/util/tr2.h =================================================================== --- mvapich/trunk/mpid/util/tr2.h 2007-12-07 15:12:36 UTC (rev 1649) +++ mvapich/trunk/mpid/util/tr2.h 2007-12-08 00:42:08 UTC (rev 1650) @@ -9,6 +9,8 @@ #include "mpichconf.h" #endif +#undef FREE + /* Define MPIR_MEMDEBUG to enable these memory tracing routines */ #if defined(MPIR_MEMDEBUG) || defined(_TR_SOURCE) From perkinjo at mvapich.cse.ohio-state.edu Mon Dec 10 11:05:47 2007 From: perkinjo at mvapich.cse.ohio-state.edu (perkinjo@mvapich.cse.ohio-state.edu) Date: Mon Dec 10 11:06:31 2007 Subject: [mvapich-commit] r1651 - mvapich/trunk/mpid/ch_gen2/process Message-ID: <200712101605.lBAG5laN009824@mvapich.cse.ohio-state.edu> Author: perkinjo Date: 2007-12-10 11:05:45 -0500 (Mon, 10 Dec 2007) New Revision: 1651 Modified: mvapich/trunk/mpid/ch_gen2/process/minidaemon.c mvapich/trunk/mpid/ch_gen2/process/minidaemon.h mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.h mvapich/trunk/mpid/ch_gen2/process/pmgr_collective_common.c Log: Bug fixes and minor enhancements. Thanks to Mellanox for the patches. Modified: mvapich/trunk/mpid/ch_gen2/process/minidaemon.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/minidaemon.c 2007-12-08 00:42:08 UTC (rev 1650) +++ mvapich/trunk/mpid/ch_gen2/process/minidaemon.c 2007-12-10 16:05:45 UTC (rev 1651) @@ -87,6 +87,7 @@ static const int recv_children_term_timeout = 30; static const int recv_jobs_term_timeout = 10; + /** These 2 structures are for temporarily testing only **/ /**-----------------------------------------------------------**/ @@ -135,6 +136,9 @@ int ppid; int isLeaf; int array_it; + + md_exit_status_type md_exit_value; + }; static struct minidaemon_t md_entity; @@ -153,7 +157,7 @@ int port; int rank; - MD_status proc_state; + int proc_state; int child_list_len; /* Only for MD nodes */ }; @@ -315,6 +319,7 @@ md_entity.mpirun_port = mpirun_port; md_entity.ppid = ppid; md_entity.isLeaf = 0; + md_entity.md_exit_value = MD_EXIT_NORMAL; } /* create parent socket and children minidaemon sockets */ @@ -332,7 +337,7 @@ if ((s = socket (AF_INET, SOCK_STREAM, 0)) < 0) { perror ("socket"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } @@ -343,13 +348,13 @@ /* determine host system name and internet address*/ if (uname(&name) == -1) { perror ("uname"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } if ( (host=gethostbyname(name.nodename)) == NULL) { MD_PRINT(DDEBUG,"The problem is here, nodename is %s\n",name.nodename); perror ("gethostbyname"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } /*memcpy ( (char*) &srv_sock_name.sin_addr, host->h_addr_list, host->h_length);*/ @@ -357,25 +362,25 @@ if (bind (s, (struct sockaddr *) &srv_sock_name, sizeof (srv_sock_name)) < 0) { perror ("bind"); close (s); - exit(1); + exit(MD_EXIT_SYS_ERROR); } /* Set non-blocking */ if( (arg = fcntl(s, F_GETFL, NULL)) == 0) { fprintf(stderr, "Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); - exit(0); + exit(MD_EXIT_SYS_ERROR); } arg |= O_NONBLOCK; if( fcntl(s, F_SETFL, arg) != 0) { fprintf(stderr, "Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); - exit(0); + exit(MD_EXIT_SYS_ERROR); } /* prepare to listen cp for multiple connections*/ if (listen (s, md_entity.tree_width) ==-1 ) { perror ("listen"); close (s); - exit(1); + exit(MD_EXIT_SYS_ERROR); } MD_PRINT(DDEBUG,"Waiting for children to connect...\n"); @@ -403,11 +408,11 @@ /* timeout was reached */ MD_PRINT(DDEBUG,"Timeout problem when connecting to children minidaemons"); md_forced_cleanup_handler(); - exit(1); + exit(MD_EXIT_MINIDAEMON_FAIL); default: if (res < 0) { perror ("Bad select result\n"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } if (FD_ISSET (s, &mask)) { //TODO : yet another try @@ -421,7 +426,7 @@ accept(s, (struct sockaddr *) &clnt_sock_name, &client_len)) == -1 ) { perror ("listen"); close (s); - exit(1); + exit(MD_EXIT_SYS_ERROR); } md_entity.max_fd_num = ( md_entity.max_fd_num > md_entity.children_md_arr[loop].fd @@ -463,8 +468,10 @@ md_general_listen(); MD_PRINT(DDEBUG,"Exiting : unhandled signal, pid=%d\n",md_entity.pid); - md_print_status_message(); - exit(0); + if (md_entity.isParent) { + md_print_status_message(); + } + exit(MD_EXIT_MINIDAEMON_SIG); } void md_connect_to_parent() { @@ -479,7 +486,7 @@ MD_PRINT(DDEBUG,"Connecting to parent : %s\n", md_entity.parent_name); if ((s = socket (AF_INET, SOCK_STREAM, 0)) < 0) { perror ("socket"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } memset ((char *)&srv_sock_name, '\0', sizeof(srv_sock_name)); @@ -487,9 +494,8 @@ srv_sock_name.sin_family = AF_INET; if ( (host=gethostbyname(md_entity.parent_name)) == NULL) { - MD_PRINT(DDEBUG,"The problem is here, parent name = %s\n",md_entity.parent_name); perror ("gethostbyname"); - exit(1); + exit(MD_EXIT_SYS_ERROR); } //TODO host->h_addr_list @@ -499,17 +505,17 @@ if (connect (s, (struct sockaddr *) &srv_sock_name, sizeof (srv_sock_name)) < 0) { perror ("connect"); close (s); - exit(1); + exit(MD_EXIT_SYS_ERROR); } /* Set non-blocking */ if( (arg = fcntl(s, F_GETFL, NULL)) == 0) { fprintf(stderr, "Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); - exit(0); + exit(MD_EXIT_SYS_ERROR); } arg |= O_NONBLOCK; if( fcntl(s, F_SETFL, arg) != 0) { fprintf(stderr, "Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); - exit(0); + exit(MD_EXIT_SYS_ERROR); } MD_PRINT(DDEBUG,"Connected to parent\n"); @@ -891,8 +897,9 @@ case -1: /* Signal or error was received */ perror(" Unhandled signal\n"); - md_print_status_message(); - exit(1); + if (md_entity.isParent) + md_print_status_message(); + exit(MD_EXIT_MINIDAEMON_SIG); case 0: /* Is parent timeout ? */ timeout_handler((int)(time(NULL) - time_wasted)); @@ -939,7 +946,7 @@ case MD_WORK : if (md_entity.send2father_timeout < md_entity.recv_children_timeout) { if (!md_entity.isParent) { - MD_PRINT(DDEBUG," Sending status to parent, pid=%d\n",md_entity.pid); + MD_PRINT(DINFO," Sending status to parent, pid=%d\n",md_entity.pid); md_send_status_message(md_entity.total_running_jobs,-1); } md_entity.send2father_timeout = send2father_work_timeout; @@ -959,6 +966,7 @@ case MD_TERM : /* here we should wait only for children process and children minidaemons */ //TODO propagation delay, i.e. to_value = f(base_to_value,tree_depth) + sleep(1); if (md_entity.total_running_jobs !=0) { md_forced_cleanup_handler(); } @@ -967,10 +975,13 @@ if (!md_entity.isParent) { md_send_status_message(md_entity.total_running_jobs,-1); } - MD_PRINT(DDEBUG,"Minidaemon exits right now, pid=%d\n",md_entity.pid); + MD_PRINT(DINFO,"Minidaemon exits right now, pid=%d\n",md_entity.pid); + if (md_entity.isParent) { + md_print_status_message(); + } + sleep(1); md_socket_close(); - md_print_status_message(); - exit(0); + exit(md_entity.md_exit_value); default: break; @@ -980,11 +991,16 @@ void md_children_proc_handler(int ind, int time_delta) { /* static int counter = -1 ; */ - int *loc_state; + int loc_state; /*md_entity.children_md_arr[ind].stat = */ - waitpid (md_entity.childrenList[ind].pid,loc_state,WNOHANG); - MD_PRINT(DDEBUG," Job %d was finished with status %d\n",md_entity.childrenList[ind].pid, WEXITSTATUS (*loc_state)); - md_entity.childrenList[ind].proc_state = (loc_state ? WEXITSTATUS (*loc_state) : -1); + waitpid (md_entity.childrenList[ind].pid,&loc_state,WNOHANG); + MD_PRINT(DDEBUG," Job %d was finished with status %d\n",md_entity.childrenList[ind].pid, WEXITSTATUS (loc_state)); + //MD_PRINT(DINFO,"The exit status of job %d is %d\n",ind,WEXITSTATUS (loc_state)); + md_entity.childrenList[ind].proc_state = (WIFEXITED(loc_state) ? WEXITSTATUS (loc_state) : -1); + MD_PRINT(DINFO,"The exit status of job %d is %d\n",ind,md_entity.childrenList[ind].proc_state); + md_entity.md_exit_value = (md_entity.childrenList[ind].proc_state == MD_EXIT_NORMAL + ? md_entity.md_exit_value : MD_EXIT_MINIDAEMON_FAIL); + --md_entity.curr_running_jobs; if ( md_entity.curr_running_jobs == 0) { /* the last job */ @@ -997,9 +1013,12 @@ if (md_entity.isParent) { md_print_status_message(); } else { - md_send_status_message(md_entity.total_running_jobs,ind /*md_entity.childrenList[ind].proc_state*/); + if (!md_entity.isParent) { + md_send_status_message(md_entity.total_running_jobs, ind); + } } - exit(0); + MD_PRINT(DINFO," All own jobs and children minidaemons finished : exiting\n"); + exit(md_entity.md_exit_value); } } if (md_entity.curr_running_jobs == (md_entity.total_running_jobs-1) ) { @@ -1013,17 +1032,24 @@ md_entity.recv_children_timeout -= time_delta; } if (!md_entity.isParent) { - md_send_status_message(md_entity.total_running_jobs, md_entity.childrenList[ind].proc_state); + md_send_status_message(md_entity.total_running_jobs, ind); } } void md_print_status_message() { int i; - MD_PRINT(DDEBUG,"Printing the status of all the processes: "); - for (i = 0; i < md_entity.ch_num ; ++i) { - fprintf(stderr,"%d ", md_entity.childrenList[i].proc_state); fflush(stderr); + if (md_entity.md_exit_value == MD_EXIT_NORMAL) { + MD_PRINT(DNONE,"All user jobs finished normally. Minidaemon will shutdown now\n"); } - fprintf(stderr,"\n"); fflush(stderr); + else { + /*MD_PRINT(DNONE,"MPI run finished,printing the exit status of all mpi jobs: ");*/ + fprintf(stderr,"One ore more user jobs finished with bad status, printing:\n"); + fflush(stderr); + for (i = 0; i < md_entity.ch_num ; ++i) { + fprintf(stderr,"%d ", md_entity.childrenList[i].proc_state); fflush(stderr); + } + fprintf(stderr,"\n"); fflush(stderr); + } } //TODO array of pointer to appropriate function , i.e. func_array[msg_buff[0]].handler(); @@ -1086,17 +1112,22 @@ if (msg_buf[array_it_index] != -1) { MD_SYS_ERROR(" : message too short\n"); } - if (i != md_entity.curr_running_jobs) { /* If parent had answered us, there's no need to send him msg right now */ + if (i != md_entity.total_running_jobs) { /* If parent had answered us, there's no need to send him msg right now */ md_send_status_message(i,-1); } return ; } assert (size == sizeof(int)*4) ; iter = msg_buf[array_it_index] + msg_buf[offset_index]; - MD_PRINT(DDEBUG," Setting status %d to index %d\n", msg_buf[proc_stat_val_index],iter); + MD_PRINT(DINFO," Setting status %d to index %d\n", msg_buf[proc_stat_val_index],iter); md_entity.childrenList[iter].proc_state = msg_buf[proc_stat_val_index]; - if (i != md_entity.curr_running_jobs) { /* If parent had answered us, there's no need to send him msg right now */ + md_entity.md_exit_value = (md_entity.childrenList[iter].proc_state == MD_EXIT_NORMAL + ? md_entity.md_exit_value : MD_EXIT_MINIDAEMON_FAIL); + if (i != md_entity.total_running_jobs) { /* If parent had answered us, there's no need to send him msg right now */ md_send_status_message(i,-1); + if (!md_entity.isParent) { + md_send_status_message(md_entity.total_running_jobs,iter); /*update the parent*/ + } } } @@ -1118,8 +1149,10 @@ buff[2] = iter; buff[3] = md_entity.childrenList[iter].proc_state; write_size = sizeof(int) * status_buff_len; + MD_PRINT(DINFO,"Sending status message to socket %d ,md_entity.array_it =%d,iter =%d, proc_state=%d\n", + sd,md_entity.array_it,iter, md_entity.childrenList[iter].proc_state); + } - MD_PRINT(DDEBUG,"Sending status message to socket %d \n",sd); if (write(sd,buff,write_size) < write_size ) { if (md_entity.phase != MD_TERM) MD_SYS_ERROR(""); @@ -1202,18 +1235,30 @@ } void md_forced_cleanup_handler() { + static const int md_killed_job_status = -2; /* send kill to its own jobs */ ChildrenListIterator i; - int *loc_state; + int loc_state; /* send forced kill (kill -9) to its own jobs */ - MD_PRINT(DDEBUG," Starting ...\n"); + MD_PRINT(DINFO," Starting ...\n"); for (i=0; i < getListNextNode(0,1) ; ++i) { - kill(md_entity.childrenList[i].pid,SIGKILL); + int res; + /* If the result was negative, the error was happen. + In General case, we shold exit, but here we must continue job termination, + in spite of the error + */ + if ((res = waitpid(md_entity.childrenList[i].pid,&loc_state,WNOHANG)) == 0) { + /* Job is still running */ + kill(md_entity.childrenList[i].pid,SIGKILL); + res = waitpid(md_entity.childrenList[i].pid,&loc_state,WNOHANG); + } + md_entity.childrenList[i].proc_state = md_killed_job_status; + if (res > 0) { + md_entity.childrenList[i].proc_state = (WIFEXITED(loc_state) ? WEXITSTATUS (loc_state) : -1); + } + md_send_status_message(md_entity.total_running_jobs,i); + MD_PRINT(DINFO,"The exit status of possibly killed job %d is %d\n",i,md_entity.childrenList[i].proc_state ); } - for (i=0; i < getListNextNode(0,1) ; ++i) { - waitpid(md_entity.childrenList[i].pid,loc_state,WNOHANG); - md_entity.childrenList[i].proc_state = (loc_state ? WEXITSTATUS (*loc_state) : -1); - } } void md_socket_close() { @@ -1287,8 +1332,10 @@ MD_PRINT(DDEBUG,"Handling SIGPIPE\n"); md_forced_cleanup_handler(); - md_print_status_message(); - exit(1); + if (md_entity.isParent) { + md_print_status_message(); + } + exit(MD_EXIT_MINIDAEMON_SIG); } static void get_display_str(char * display) { Modified: mvapich/trunk/mpid/ch_gen2/process/minidaemon.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/minidaemon.h 2007-12-08 00:42:08 UTC (rev 1650) +++ mvapich/trunk/mpid/ch_gen2/process/minidaemon.h 2007-12-10 16:05:45 UTC (rev 1651) @@ -18,8 +18,9 @@ additional management and resources **/ -/*#include */ #include "mpirun_rsh.h" + + #define MD_USR_ERROR(msg) \ do { \ fprintf(stderr,"%s\n",msg); \ @@ -29,9 +30,18 @@ #define MD_SYS_ERROR(msg) \ do { \ perror(msg); \ - exit(1); \ + exit(MD_EXIT_SYS_ERROR); \ } while (0); +/* Minidaemon exit statuses */ +typedef enum { + MD_EXIT_NORMAL = 0, /* use this exit status when all user apllications finished successfully */ + MD_EXIT_SYS_ERROR = 1, /* use this exit status when system error was happen */ + MD_EXIT_APP_FAIL = -1, /* use this exit status when user application(s) exited with bad status */ + MD_EXIT_MINIDAEMON_FAIL = -2, /* use this exit status when md failed or there was timeout on connection */ + MD_EXIT_MINIDAEMON_SIG = -3 /* use this exit status when md failed or there was timeout on connection */ +} md_exit_status_type; + /* Support for debug prints. */ /*#define DNONE 0*/ /* Message with this debug level will be always printed */ Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2007-12-08 00:42:08 UTC (rev 1650) +++ mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.c 2007-12-10 16:05:45 UTC (rev 1651) @@ -57,7 +57,7 @@ */ #include "mpirun_rsh.h" - +#include "minidaemon.h" process_groups * pglist = NULL; process * plist = NULL; int nprocs = 0; @@ -448,9 +448,9 @@ alarm(1000); alarm_msg = "Timeout during client startup.\n"; /* long timeout for testing, where process may be stopped in debugger */ - + int md_id; if (use_xlauncher) { - int md_id = fork(); + md_id = fork(); if (md_id == 0) { char command_name[COMMAND_LEN]; char command_name_tv[COMMAND_LEN]; @@ -471,6 +471,11 @@ minidaemon_create(plist,nprocs,xlauncher_width,"VIADEV_PARAM=DEMO_PARAM",command_name, port, getpid()); minidaemon_run(); + /* minidaemon should not reach this line in normal flow, + only special signal can be the reason */ + MD_PRINT(DNONE,"Finished minidaemon,exiting\n"); + exit(MD_EXIT_MINIDAEMON_SIG); + } } @@ -562,6 +567,13 @@ plist[i].state = P_RUNNING; } + if(use_xlauncher) { + int status; + + waitpid(md_id, &status, 0); + exit(WIFEXITED(status) ? WEXITSTATUS (status) : MD_EXIT_MINIDAEMON_SIG); + } + wait_for_errors(s, (struct sockaddr *)&sockaddr, sockaddr_len); /* Modified: mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.h =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.h 2007-12-08 00:42:08 UTC (rev 1650) +++ mvapich/trunk/mpid/ch_gen2/process/mpirun_rsh.h 2007-12-10 16:05:45 UTC (rev 1651) @@ -88,13 +88,20 @@ #define DDEBUG 2 /* Debug level, should be turned off in normal run */ #define DPATH 3 /* The highest level, for use in loops or critical sections */ -#define DGLOBAL_LEVEL 3 +#ifndef DGLOBAL_LEVEL +#define DGLOBAL_LEVEL 0 +#endif /* DGLOBAL_LEVEL should be defined in compilation stage, otherwise it should be zero */ +#if DGLOBAL_LEVEL >= 0 #define MD_PRINT(dlevel,fmt, args...) {if (dlevel <= DGLOBAL_LEVEL){\ fprintf(stderr, "[%s:%d, pid=%d]", __FILE__, __LINE__,getpid());\ fprintf(stderr, fmt, ## args); fflush(stderr);}} +#else +#define MD_PRINT(dlevel,fmt, args...) +#endif + typedef enum { P_NOTSTARTED, P_STARTED, Modified: mvapich/trunk/mpid/ch_gen2/process/pmgr_collective_common.c =================================================================== --- mvapich/trunk/mpid/ch_gen2/process/pmgr_collective_common.c 2007-12-08 00:42:08 UTC (rev 1650) +++ mvapich/trunk/mpid/ch_gen2/process/pmgr_collective_common.c 2007-12-10 16:05:45 UTC (rev 1651) @@ -45,7 +45,7 @@ rc = write(fd, offset, size - n); if(rc < 0) { - if(errno == EINTR) continue; + if(errno == EINTR || errno == EAGAIN) continue; return rc; } @@ -71,7 +71,7 @@ rc = read(fd, offset, size - n); if(rc < 0) { - if(errno == EINTR) continue; + if(errno == EINTR || errno == EAGAIN) continue; return rc; } From curtisbr at mvapich.cse.ohio-state.edu Mon Dec 10 16:06:53 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Mon Dec 10 16:07:38 2007 Subject: [mvapich-commit] r1652 - mvapich2/trunk/src/mpid/osu_ch3/src Message-ID: <200712102106.lBAL6rxH010230@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-10 16:06:51 -0500 (Mon, 10 Dec 2007) New Revision: 1652 Modified: mvapich2/trunk/src/mpid/osu_ch3/src/ch3u_handle_connection.c Log: Corrections for Sun Studio compiler warnings. Modified: mvapich2/trunk/src/mpid/osu_ch3/src/ch3u_handle_connection.c =================================================================== --- mvapich2/trunk/src/mpid/osu_ch3/src/ch3u_handle_connection.c 2007-12-10 16:05:45 UTC (rev 1651) +++ mvapich2/trunk/src/mpid/osu_ch3/src/ch3u_handle_connection.c 2007-12-10 21:06:51 UTC (rev 1652) @@ -300,7 +300,7 @@ { /* FIXME: Debugging */ if (vc->state != MPIDI_VC_STATE_ACTIVE) { - printf("Unexpected state %d in vc %x\n", vc->state, (int)vc); + printf("Unexpected state %d in vc %x\n", vc->state, (unsigned long)vc); fflush(stdout); } MPIU_DBG_MSG_D(CH3_DISCONNECT,VERBOSE, From curtisbr at mvapich.cse.ohio-state.edu Mon Dec 10 16:07:17 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Mon Dec 10 16:07:58 2007 Subject: [mvapich-commit] r1653 - mvapich2/trunk/src/binding/f77 Message-ID: <200712102107.lBAL7H7m010240@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-10 16:07:17 -0500 (Mon, 10 Dec 2007) New Revision: 1653 Modified: mvapich2/trunk/src/binding/f77/filecallerrf.c mvapich2/trunk/src/binding/f77/filegeterrf.c mvapich2/trunk/src/binding/f77/fileseterrf.c Log: Corrections for Sun Studio compiler warnings. Modified: mvapich2/trunk/src/binding/f77/filecallerrf.c =================================================================== --- mvapich2/trunk/src/binding/f77/filecallerrf.c 2007-12-10 21:06:51 UTC (rev 1652) +++ mvapich2/trunk/src/binding/f77/filecallerrf.c 2007-12-10 21:07:17 UTC (rev 1653) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_call_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_call_errhandler( MPI_File_f2c(*v1), *v2 ); + *ierr = MPI_File_call_errhandler( *(MPI_File_f2c(v1)), *v2 ); } Modified: mvapich2/trunk/src/binding/f77/filegeterrf.c =================================================================== --- mvapich2/trunk/src/binding/f77/filegeterrf.c 2007-12-10 21:06:51 UTC (rev 1652) +++ mvapich2/trunk/src/binding/f77/filegeterrf.c 2007-12-10 21:07:17 UTC (rev 1653) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_get_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_get_errhandler( MPI_File_f2c(*v1), v2 ); + *ierr = MPI_File_get_errhandler( *(MPI_File_f2c(v1)), v2 ); } Modified: mvapich2/trunk/src/binding/f77/fileseterrf.c =================================================================== --- mvapich2/trunk/src/binding/f77/fileseterrf.c 2007-12-10 21:06:51 UTC (rev 1652) +++ mvapich2/trunk/src/binding/f77/fileseterrf.c 2007-12-10 21:07:17 UTC (rev 1653) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_set_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_set_errhandler( MPI_File_f2c(*v1), *v2 ); + *ierr = MPI_File_set_errhandler( *(MPI_File_f2c(v1)), *v2 ); } From koop at mvapich.cse.ohio-state.edu Tue Dec 11 10:21:55 2007 From: koop at mvapich.cse.ohio-state.edu (koop@mvapich.cse.ohio-state.edu) Date: Tue Dec 11 10:22:38 2007 Subject: [mvapich-commit] r1654 - mvapich/trunk/mpid/ch_gen2_ud Message-ID: <200712111521.lBBFLtWW012666@mvapich.cse.ohio-state.edu> Author: koop Date: 2007-12-11 10:21:53 -0500 (Tue, 11 Dec 2007) New Revision: 1654 Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_buf.c Log: * Fix typo in mv_rbuf_size allocation. Given current code flow no error cases have occurred. (Thanks to Alexei Adamovich for pointing this out) Modified: mvapich/trunk/mpid/ch_gen2_ud/mv_buf.c =================================================================== --- mvapich/trunk/mpid/ch_gen2_ud/mv_buf.c 2007-12-10 21:07:17 UTC (rev 1653) +++ mvapich/trunk/mpid/ch_gen2_ud/mv_buf.c 2007-12-11 15:21:53 UTC (rev 1654) @@ -112,7 +112,7 @@ if(!found) { /* TODO: re-order based on size!!! */ - b = &(mv_rbuf_avail[mv_sbuf_avail_num]); + b = &(mv_rbuf_avail[mv_rbuf_avail_num]); b->alloc_size = rbuf_size; /* TODO: change if using headers */ From curtisbr at mvapich.cse.ohio-state.edu Tue Dec 11 16:57:58 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Tue Dec 11 16:58:40 2007 Subject: [mvapich-commit] r1655 - mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2 Message-ID: <200712112157.lBBLvwqK013392@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-11 16:57:57 -0500 (Tue, 11 Dec 2007) New Revision: 1655 Added: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_impl.h Log: Corrections for warnings from the Sun Studio 12 compiler). Added: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_impl.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_impl.h 2007-12-11 15:21:53 UTC (rev 1654) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_impl.h 2007-12-11 21:57:57 UTC (rev 1655) @@ -0,0 +1,12 @@ +#ifndef IBV_IMPL_H_ +#define IBV_IMPL_H_ + +void adjust_weights(MPIDI_VC_t *vc, double start_time, + double *finish_time, + double *init_weight); + +int get_wall_time(double *t); + +int perform_manual_apm(struct ibv_qp* qp); + +#endif /* IBV_IMPL_H_ */ From curtisbr at mvapich.cse.ohio-state.edu Tue Dec 11 16:58:43 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Tue Dec 11 16:59:25 2007 Subject: [mvapich-commit] r1656 - in mvapich2/branches/1.0/src: binding/f77 include mpid/osu_ch3/channels/mrail/include mpid/osu_ch3/channels/mrail/src/gen2 mpid/osu_ch3/channels/mrail/src/rdma mpid/osu_ch3/channels/mrail/src/udapl mpid/osu_ch3/channels/mrail/src/vapi mpid/osu_ch3/src pmi/simple Message-ID: <200712112158.lBBLwhOb013402@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-11 16:58:43 -0500 (Tue, 11 Dec 2007) New Revision: 1656 Modified: mvapich2/branches/1.0/src/binding/f77/filecallerrf.c mvapich2/branches/1.0/src/binding/f77/filegeterrf.c mvapich2/branches/1.0/src/binding/f77/fileseterrf.c mvapich2/branches/1.0/src/include/pmi.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/include/mpidi_ch3_impl.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_priv.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mpidi_ch3_rdma_post.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/rdma/ch3_smp_progress.c mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/udapl/mpidi_ch3_rdma_post.h mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/vapi/mpidi_ch3_rdma_post.h mvapich2/branches/1.0/src/mpid/osu_ch3/src/ch3u_handle_connection.c mvapich2/branches/1.0/src/pmi/simple/simple_pmi.c Log: Corrections for warnings from the Sun Studio 12 compiler). Modified: mvapich2/branches/1.0/src/binding/f77/filecallerrf.c =================================================================== --- mvapich2/branches/1.0/src/binding/f77/filecallerrf.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/binding/f77/filecallerrf.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_call_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_call_errhandler( MPI_File_f2c(*v1), *v2 ); + *ierr = MPI_File_call_errhandler( *(MPI_File_f2c(v1)), *v2 ); } Modified: mvapich2/branches/1.0/src/binding/f77/filegeterrf.c =================================================================== --- mvapich2/branches/1.0/src/binding/f77/filegeterrf.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/binding/f77/filegeterrf.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_get_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_get_errhandler( MPI_File_f2c(*v1), v2 ); + *ierr = MPI_File_get_errhandler( *(MPI_File_f2c(v1)), v2 ); } Modified: mvapich2/branches/1.0/src/binding/f77/fileseterrf.c =================================================================== --- mvapich2/branches/1.0/src/binding/f77/fileseterrf.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/binding/f77/fileseterrf.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -120,5 +120,5 @@ /* Prototypes for the Fortran interfaces */ #include "fproto.h" FORT_DLL_SPEC void FORT_CALL mpi_file_set_errhandler_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){ - *ierr = MPI_File_set_errhandler( MPI_File_f2c(*v1), *v2 ); + *ierr = MPI_File_set_errhandler( *(MPI_File_f2c(v1)), *v2 ); } Modified: mvapich2/branches/1.0/src/include/pmi.h =================================================================== --- mvapich2/branches/1.0/src/include/pmi.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/include/pmi.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -737,7 +737,7 @@ allocated with 'malloc()'. @*/ -int PMI_Args_to_keyval(int *argcp, char *((*argvp)[]), PMI_keyval_t **keyvalp, int *size); +int PMI_Args_to_keyval(int *argcp, char ***argvp, PMI_keyval_t **keyvalp, int *size); /*@ PMI_Free_keyvals - free the keyval structures created by PMI_Args_to_keyval Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/include/mpidi_ch3_impl.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/include/mpidi_ch3_impl.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/include/mpidi_ch3_impl.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -427,8 +427,23 @@ int MPIDI_CH3I_SMP_finalize(void); +int MPIDI_CH3I_SMP_writev_rndv_header(MPIDI_VC_t * vc, const MPID_IOV * iov, + const int n, int *num_bytes_ptr); + +int MPIDI_CH3I_SMP_writev_rndv_data_cont(MPIDI_VC_t * vc, const MPID_IOV * iov, + const int n, int *num_bytes_ptr); + +int MPIDI_CH3I_SMP_writev_rndv_data(MPIDI_VC_t * vc, const MPID_IOV * iov, + const int n, int *num_bytes_ptr); + int MPIDI_CH3I_SMP_writev(MPIDI_VC_t * vc, const MPID_IOV * iov, const int n, int *num_bytes_ptr); + +int MPIDI_CH3I_SMP_readv_rndv_cont(MPIDI_VC_t * recv_vc_ptr, const MPID_IOV * iov, + const int iovlen, int index, int *num_bytes_ptr); + +int MPIDI_CH3I_SMP_readv_rndv(MPIDI_VC_t * recv_vc_ptr, const MPID_IOV * iov, + const int iovlen, int index, int *num_bytes_ptr); int MPIDI_CH3I_SMP_readv(MPIDI_VC_t * recv_vc_ptr, const MPID_IOV * iov, const int iovlen, int @@ -443,6 +458,14 @@ int fd; }; +int MPIDI_CH3I_SHMEM_COLL_init(MPIDI_PG_t *pg); + +int MPIDI_CH3I_SHMEM_COLL_Mmap(); + +int MPIDI_CH3I_SHMEM_COLL_finalize(); + +void MPIDI_CH3I_SHMEM_COLL_Unlink(); + #endif /********* End of OSU-MPI2 *************************/ Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_priv.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_priv.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_priv.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -110,7 +110,7 @@ v = ((vbuf *)vbuf_ctrl_buf) + i; v->head_flag = (VBUF_FLAG_TYPE *) ( (char *)(vbuf_rdma_buf) + (i + 1) * rdma_vbuf_total_size - sizeof *v->head_flag); - v->buffer = (char *) ( (char *)(vbuf_rdma_buf) + (i * + v->buffer = (unsigned char *) ( (char *)(vbuf_rdma_buf) + (i * rdma_vbuf_total_size) ); v->vc = c; } Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -11,6 +11,7 @@ */ #include "rdma_impl.h" +#include "ibv_impl.h" #include "vbuf.h" #include "dreg.h" @@ -27,6 +28,8 @@ #define DEBUG_PRINT(args...) #endif +void get_sorted_index(MPIDI_VC_t *vc, int *b); + int MPIDI_CH3I_MRAIL_Prepare_rndv(MPIDI_VC_t * vc, MPID_Request * req) { dreg_entry *reg_entry; @@ -681,7 +684,7 @@ /* Get the sorted indices for the given array */ -get_sorted_index(MPIDI_VC_t *vc, int *b) +void get_sorted_index(MPIDI_VC_t *vc, int *b) { int *taken; Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -17,6 +17,7 @@ */ #include "rdma_impl.h" +#include "ibv_impl.h" #include "vbuf.h" #include "pmi.h" Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -22,18 +22,15 @@ static void set_real_munmap_ptr() { - char *dlerror_str; - void *ptr_dlsym; + munmap_t munmap = (munmap_t) dlsym(RTLD_NEXT, "munmap"); + char* dlerror_str = dlerror(); - ptr_dlsym = dlsym(RTLD_NEXT, "munmap"); - dlerror_str = dlerror(); - if(NULL != dlerror_str) { fprintf(stderr,"Error resolving munmap (%s)\n", dlerror_str); } - mvapich2_minfo.munmap = ptr_dlsym; + mvapich2_minfo.munmap = munmap; } #endif Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -21,6 +21,10 @@ #include "ptmalloc2/malloc.h" #include "ptmalloc2/sysdeps/pthread/malloc-machine.h" +#ifndef DISABLE_MUNMAP_HOOK +typedef int (*munmap_t)(void*, size_t); +#endif /* ifndef DISABLE_MUNMAP_HOOK */ + typedef struct { int is_our_malloc; int is_our_free; @@ -31,7 +35,7 @@ int is_inside_free; int is_mem_hook_finalized; #ifndef DISABLE_MUNMAP_HOOK - int (*munmap)(void*, size_t); + munmap_t munmap; #endif } mvapich2_malloc_info_t; Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mpidi_ch3_rdma_post.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mpidi_ch3_rdma_post.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/mpidi_ch3_rdma_post.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -216,11 +216,16 @@ int MPIDI_CH3I_MRAILI_Get_next_vbuf_local(MPIDI_VC_t *vc, vbuf ** vbuf_handle, int is_blocking); + +int MPIDI_CH3I_MRAILI_Get_next_vbuf(MPIDI_VC_t ** vc_ptr, + vbuf ** vbuf_ptr, int is_blocking); int MPIDI_CH3I_MRAILI_Waiting_msg(MPIDI_VC_t * vc, vbuf **, int blocking); int MPIDI_CH3I_MRAILI_Cq_poll(vbuf **, MPIDI_VC_t *, int, int); +void MRAILI_Send_noop(MPIDI_VC_t * c, int rail); + int MRAILI_Send_noop_if_needed(MPIDI_VC_t *vc, int rail); int MRAILI_Send_rdma_credit_if_needed(MPIDI_VC_t *vc); Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -402,4 +402,13 @@ uint16_t get_pkey_index(uint16_t pkey, int hca_num, int port_num); void set_pkey_index(uint16_t * pkey_index, int hca_num, int port_num); +void init_apm_lock(); + +void MRAILI_RDMA_Get_finish(MPIDI_VC_t * vc, + MPID_Request * rreq, int rail); + +int reload_alternate_path(struct ibv_qp *qp); + +int power_two(int x); + #endif /* RDMA_IMPL_H */ Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -194,7 +194,7 @@ cur->region = reg; cur->head_flag = (VBUF_FLAG_TYPE *) ((char *)(vbuf_dma_buffer) + (i + 1) * rdma_vbuf_total_size - sizeof *cur->head_flag); - cur->buffer = (char *) ((char *)(vbuf_dma_buffer) + + cur->buffer = (unsigned char *) ((char *)(vbuf_dma_buffer) + (i * rdma_vbuf_total_size)); cur->eager = 0; @@ -210,7 +210,7 @@ cur->head_flag = (VBUF_FLAG_TYPE *) ((char *)vbuf_dma_buffer + (nvbufs * rdma_vbuf_total_size) - sizeof *cur->head_flag); - cur->buffer = (char *) ((char *)vbuf_dma_buffer + + cur->buffer = (unsigned char *) ((char *)vbuf_dma_buffer + ((nvbufs - 1) * rdma_vbuf_total_size)); cur->eager = 0; cur->content_size = 0; Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/rdma/ch3_smp_progress.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/rdma/ch3_smp_progress.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/rdma/ch3_smp_progress.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -960,8 +960,7 @@ if (enable_shmem_collectives){ /* Unlinking shared files for collectives*/ - mpi_errno = MPIDI_CH3I_SHMEM_COLL_Unlink(); - if(mpi_errno) MPIU_ERR_POP(mpi_errno); + MPIDI_CH3I_SHMEM_COLL_Unlink(); } pid = getpid(); @@ -2407,7 +2406,7 @@ if (1 == pg_size) hostnames_j[0] = hostid; /* smpi.local_nodes = (unsigned int *) malloc(pg_size * sizeof(int)); */ - smpi_ptr = (unsigned int *) malloc(pg_size * sizeof(int)); + smpi_ptr = (int *) malloc(pg_size * sizeof(int)); if(smpi_ptr == NULL) { MPIU_ERR_SETFATALANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem", Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/udapl/mpidi_ch3_rdma_post.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/udapl/mpidi_ch3_rdma_post.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/udapl/mpidi_ch3_rdma_post.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -201,11 +201,15 @@ int MPIDI_CH3I_MRAILI_Get_next_vbuf_local (MPIDI_VC_t * vc, vbuf ** vbuf_handle); + +int MPIDI_CH3I_MRAILI_Get_next_vbuf(MPIDI_VC_t **vc_pptr, vbuf **v_ptr); int MPIDI_CH3I_MRAILI_Waiting_msg(MPIDI_VC_t * vc, vbuf **, int); int MPIDI_CH3I_MRAILI_Cq_poll (vbuf **, MPIDI_VC_t *, int, int); +void MRAILI_Send_noop(MPIDI_VC_t * c, const MRAILI_Channel_info * channel); + int MRAILI_Send_noop_if_needed (MPIDI_VC_t * vc, const MRAILI_Channel_info * channel); Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/vapi/mpidi_ch3_rdma_post.h =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/vapi/mpidi_ch3_rdma_post.h 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/channels/mrail/src/vapi/mpidi_ch3_rdma_post.h 2007-12-11 21:58:43 UTC (rev 1656) @@ -182,10 +182,14 @@ int MPIDI_CH3I_MRAILI_Get_next_vbuf_local(MPIDI_VC_t *vc, vbuf ** vbuf_handle); +int MPIDI_CH3I_MRAILI_Get_next_vbuf(MPIDI_VC_t **vc_pptr, vbuf **v_ptr); + int MPIDI_CH3I_MRAILI_Waiting_msg(MPIDI_VC_t * vc, vbuf **, int); int MPIDI_CH3I_MRAILI_Cq_poll(vbuf **, MPIDI_VC_t *, int, int is_blocking); +void MRAILI_Send_noop(MPIDI_VC_t * c, const MRAILI_Channel_info * channel); + int MRAILI_Send_noop_if_needed(MPIDI_VC_t *vc, const MRAILI_Channel_info *channel); Modified: mvapich2/branches/1.0/src/mpid/osu_ch3/src/ch3u_handle_connection.c =================================================================== --- mvapich2/branches/1.0/src/mpid/osu_ch3/src/ch3u_handle_connection.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/mpid/osu_ch3/src/ch3u_handle_connection.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -300,7 +300,7 @@ { /* FIXME: Debugging */ if (vc->state != MPIDI_VC_STATE_ACTIVE) { - printf("Unexpected state %d in vc %x\n", vc->state, (int)vc); + printf("Unexpected state %d in vc %x\n", vc->state, (unsigned long)vc); fflush(stdout); } MPIU_DBG_MSG_D(CH3_DISCONNECT,VERBOSE, Modified: mvapich2/branches/1.0/src/pmi/simple/simple_pmi.c =================================================================== --- mvapich2/branches/1.0/src/pmi/simple/simple_pmi.c 2007-12-11 21:57:57 UTC (rev 1655) +++ mvapich2/branches/1.0/src/pmi/simple/simple_pmi.c 2007-12-11 21:58:43 UTC (rev 1656) @@ -781,7 +781,7 @@ return( 0 ); } -int PMI_Args_to_keyval(int *argcp, char *((*argvp)[]), PMI_keyval_t **keyvalp, +int PMI_Args_to_keyval(int *argcp, char ***argvp, PMI_keyval_t **keyvalp, int *size) { return ( 0 ); From curtisbr at mvapich.cse.ohio-state.edu Tue Dec 11 17:08:25 2007 From: curtisbr at mvapich.cse.ohio-state.edu (curtisbr@mvapich.cse.ohio-state.edu) Date: Tue Dec 11 17:20:01 2007 Subject: [mvapich-commit] r1657 - in mvapich2/trunk: maint src/include src/mpid/osu_ch3/channels/mrail/include src/mpid/osu_ch3/channels/mrail/src/gen2 src/mpid/osu_ch3/channels/mrail/src/rdma src/mpid/osu_ch3/channels/mrail/src/udapl src/mpid/osu_ch3/channels/mrail/src/vapi src/pmi/simple Message-ID: <200712112208.lBBM8PMG013443@mvapich.cse.ohio-state.edu> Author: curtisbr Date: 2007-12-11 17:08:23 -0500 (Tue, 11 Dec 2007) New Revision: 1657 Added: mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_impl.h Modified: mvapich2/trunk/maint/config.log mvapich2/trunk/maint/config.status mvapich2/trunk/maint/createcoverage mvapich2/trunk/maint/getcoverage mvapich2/trunk/src/include/pmi.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/include/mpidi_ch3_impl.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_priv.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_rndv.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/ibv_send.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/mem_hooks.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/mpidi_ch3_rdma_post.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/rdma_impl.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/gen2/vbuf.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/rdma/ch3_smp_progress.c mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/udapl/mpidi_ch3_rdma_post.h mvapich2/trunk/src/mpid/osu_ch3/channels/mrail/src/vapi/mpidi_ch3_rdma_post.h mvapich2/trunk/src/pmi/simple/simple_pmi.c Log: Corrections for warnings from the Sun Studio 12 compiler. Modified: mvapich2/trunk/maint/config.log =================================================================== --- mvapich2/trunk/maint/config.log 2007-12-11 21:58:43 UTC (rev 1656) +++ mvapich2/trunk/maint/config.log 2007-12-11 22:08:23 UTC (rev 1657) @@ -10,5778 +10,29 @@ ## Platform. ## ## --------- ## -hostname = shakey.mcs.anl.gov -uname -m = i686 -uname -r = 2.4.30-mcs-x86 +hostname = t11 +uname -m = x86_64 +uname -r = 2.6.20-2943.fc7xen uname -s = Linux -uname -v = #1 SMP Thu Apr 14 16:10:15 CDT 2005 +uname -v = #1 SMP Tue Oct 23 14:58:12 EDT 2007 /usr/bin/uname -p = unknown /bin/uname -X = unknown -/bin/arch = i686 +/bin/arch = x86_64 /usr/bin/arch -k = unknown /usr/convex/getsysinfo = unknown -hostinfo = 203intel.mcs.anl.gov -203intel2.mcs.anl.gov -241-11-cph.mcs.anl.gov -241-8-cph.mcs.anl.gov -241-9-cph.mcs.anl.gov -2623-ddu-1.mcs.anl.gov -a216-display.mcs.anl.gov -a216-video.mcs.anl.gov -a261-audio.mcs.anl.gov -a261-control.mcs.anl.gov -a261-display.mcs.anl.gov -a261-video.mcs.anl.gov -a31ploaner.mcs.anl.gov -accordion.mcs.anl.gov -adc.mcs.anl.gov -aditya.mcs.anl.gov -ag-tech.mcs.anl.gov -ag2-test.mcs.anl.gov -agony-cph.mcs.anl.gov -aha.mcs.anl.gov -ahmedmac-cph.mcs.anl.gov -aiken-laptop.mcs.anl.gov -ajc.mcs.anl.gov -akhenaten.mcs.anl.gov -akita.mcs.anl.gov -albinoni.mcs.anl.gov -alcf-plotter.mcs.anl.gov -allcock-x31.mcs.anl.gov -allegri.mcs.anl.gov -alone.mcs.anl.gov -altai.mcs.anl.gov -ambigram.mcs.anl.gov -anitescu-a31p.mcs.anl.gov -anitescu-desktop.mcs.anl.gov -anitescu-x40.mcs.anl.gov -ankh-morpork.mcs.anl.gov -anlcbq.mcs.anl.gov -anno-1.nmpdr.org -anno-2.nmpdr.org -anno-3.nmpdr.org -anodyne.mcs.anl.gov -antares.mcs.anl.gov -antireo.mcs.anl.gov -apc-k26.mcs.anl.gov -apc-l26.mcs.anl.gov -apc-lmr-1d.mcs.anl.gov -apc-lmr.mcs.anl.gov -apc-m23.mcs.anl.gov -apc-m26.mcs.anl.gov -apc-n23.mcs.anl.gov -apc-n26.mcs.anl.gov -apc-o23.mcs.anl.gov -apc-o26.mcs.anl.gov -apc-p23.mcs.anl.gov -apc-p26.mcs.anl.gov -apc-q26.mcs.anl.gov -apc-r26.mcs.anl.gov -apc-r29.mcs.anl.gov -apc00.mcs.anl.gov -apc01.mcs.anl.gov -apc02.mcs.anl.gov -apc03.mcs.anl.gov -apc04.mcs.anl.gov -apc05.mcs.anl.gov -apc06.mcs.anl.gov -apc07.mcs.anl.gov -apc08.mcs.anl.gov -apc09.mcs.anl.gov -apc10.mcs.anl.gov -apollo-i.mcs.anl.gov -apollo.mcs.anl.gov -apsextscan1.aps.anl-external.org -apu.mcs.anl.gov -arbat.mcs.anl.gov -archive.nsf-middleware.org -aries.mcs.anl.gov -arno.mcs.anl.gov -arrow.mcs.anl.gov -aslan.mcs.anl.gov -asterix.mcs.anl.gov -aswan-cph.mcs.anl.gov -atlantis.mcs.anl.gov -autoharp.mcs.anl.gov -avison.mcs.anl.gov -avrell.mcs.anl.gov -axis.mcs.anl.gov -bach.mcs.anl.gov -bacillus.minibsub.org -backup.kbt.mcs.anl.gov -backup.uc.teragrid.org -bacon.mcs.anl.gov -bair.mcs.anl.gov -balo.mcs.anl.gov -balor.mcs.anl.gov -barracuda-xp-vir.mcs.anl.gov -barracuda.mcs.anl.gov -barrada.mcs.anl.gov -barrysmith.mcs.anl.gov -barsoom.mcs.anl.gov -beaconserver.accessgrid.org -beauty.mcs.anl.gov -beisenbe-desktop.mcs.anl.gov -beisenbe-laptop.mcs.anl.gov -beluga.mcs.anl.gov -bender.mcs.anl.gov -benderxp.mcs.anl.gov -beraneklapect.mcs.anl-external.org -berry.mcs.anl.gov -bgservice.bgl.mcs.anl.gov -bio-debian.mcs.anl.gov -bio-float-1.mcs.anl.gov -bio-float-2.mcs.anl.gov -bio-float-3.mcs.anl.gov -bio-float-4.mcs.anl.gov -bio-intel-1.mcs.anl.gov -bio-intel-2.mcs.anl.gov -bio-pig.mcs.anl.gov -bio-ppc-1.mcs.anl.gov -bio-ppc-10.mcs.anl.gov -bio-ppc-11.mcs.anl.gov -bio-ppc-12.mcs.anl.gov -bio-ppc-13.mcs.anl.gov -bio-ppc-14.mcs.anl.gov -bio-ppc-15.mcs.anl.gov -bio-ppc-16.mcs.anl.gov -bio-ppc-17.mcs.anl.gov -bio-ppc-18.mcs.anl.gov -bio-ppc-19.mcs.anl.gov -bio-ppc-2.mcs.anl.gov -bio-ppc-20.mcs.anl.gov -bio-ppc-21.mcs.anl.gov -bio-ppc-22.mcs.anl.gov -bio-ppc-23.mcs.anl.gov -bio-ppc-24.mcs.anl.gov -bio-ppc-25.mcs.anl.gov -bio-ppc-26.mcs.anl.gov -bio-ppc-27.mcs.anl.gov -bio-ppc-28.mcs.anl.gov -bio-ppc-29.mcs.anl.gov -bio-ppc-3.mcs.anl.gov -bio-ppc-30.mcs.anl.gov -bio-ppc-31.mcs.anl.gov -bio-ppc-32.mcs.anl.gov -bio-ppc-33.mcs.anl.gov -bio-ppc-34.mcs.anl.gov -bio-ppc-35.mcs.anl.gov -bio-ppc-36.mcs.anl.gov -bio-ppc-37.mcs.anl.gov -bio-ppc-38.mcs.anl.gov -bio-ppc-39.mcs.anl.gov -bio-ppc-4.mcs.anl.gov -bio-ppc-40.mcs.anl.gov -bio-ppc-41.mcs.anl.gov -bio-ppc-42.mcs.anl.gov -bio-ppc-43.mcs.anl.gov -bio-ppc-44.mcs.anl.gov -bio-ppc-45.mcs.anl.gov -bio-ppc-5.mcs.anl.gov -bio-ppc-6.mcs.anl.gov -bio-ppc-7.mcs.anl.gov -bio-ppc-8.mcs.anl.gov -bio-ppc-9.mcs.anl.gov -bio-ppc-head-1.mcs.anl.gov -bio-ppc-head-2.mcs.anl.gov -bio-ppc-head-3.mcs.anl.gov -bio-ppc-head-4.mcs.anl.gov -biocvs.mcs.anl.gov -biofiler.mcs.anl.gov -biologin-1.mcs.anl.gov -biologin-2.mcs.anl.gov -biologin-3.mcs.anl.gov -bioseed.mcs.anl.gov -biosge.mcs.anl.gov -biosql.mcs.anl.gov -birdo.mcs.anl.gov -blabla.mcs.anl.gov -blackhole.mcs.anl.gov -blin.mcs.anl.gov -bliss-cph.mcs.anl.gov -blue-in-green.mcs.anl.gov -blue.mcs.anl.gov -bluebird.mcs.anl.gov -bluewave.mcs.anl.gov -bluezen.mcs.anl.gov -bluezenxp.mcs.anl.gov -bmrtemp.mcs.anl.gov -bobcat.mcs.anl.gov -bobo.mcs.anl.gov -boda-laptop.mcs.anl.gov -bodum.mcs.anl.gov -bonus.mcs.anl.gov -boomerang.mcs.anl.gov -bosporus.mcs.anl.gov -bowser.mcs.anl.gov -boxing.mcs.anl.gov -brak.mcs.anl.gov -braun.mcs.anl.gov -bravado.mcs.anl.gov -bravo.mcs.anl.gov -brooks-laptop.mcs.anl.gov -bruce.mcs.anl.gov -brumel.mcs.anl.gov -bsmith.mcs.anl.gov -bubbles.mcs.anl.gov -bugzilla.mcs.anl.gov -buntinas.mcs.anl.gov -burn.mcs.anl.gov -buzz.mcs.anl.gov -c130intel.mcs.anl.gov -c400.mcs.anl.gov -ca.globus.org -caccini.mcs.anl.gov -cairo-cph.mcs.anl.gov -candyfloss.mcs.anl.gov -cardkey.mcs.anl.gov -caribou.mcs.anl.gov -cas.mcs.anl.gov -castanet.mcs.anl.gov -cat3550.mcs.anl.gov -cave1.mcs.anl.gov -cave3.mcs.anl.gov -cave4.mcs.anl.gov -ccclos1.mcs.anl.gov -ccclos2.mcs.anl.gov -ccclos3.mcs.anl.gov -ccclos4.mcs.anl.gov -ccclos5.mcs.anl.gov -ccfs1.mcs.anl.gov -ccfs2.mcs.anl.gov -ccfs3.mcs.anl.gov -cclogin1.mcs.anl.gov -cclogin2.mcs.anl.gov -cclogin3.mcs.anl.gov -cclogin4.mcs.anl.gov -ccn1.mcs.anl.gov -ccn10.mcs.anl.gov -ccn100.mcs.anl.gov -ccn101.mcs.anl.gov -ccn102.mcs.anl.gov -ccn103.mcs.anl.gov -ccn104.mcs.anl.gov -ccn105.mcs.anl.gov -ccn106.mcs.anl.gov -ccn107.mcs.anl.gov -ccn108.mcs.anl.gov -ccn109.mcs.anl.gov -ccn11.mcs.anl.gov -ccn110.mcs.anl.gov -ccn111.mcs.anl.gov -ccn112.mcs.anl.gov -ccn113.mcs.anl.gov -ccn114.mcs.anl.gov -ccn115.mcs.anl.gov -ccn116.mcs.anl.gov -ccn117.mcs.anl.gov -ccn118.mcs.anl.gov -ccn119.mcs.anl.gov -ccn12.mcs.anl.gov -ccn120.mcs.anl.gov -ccn121.mcs.anl.gov -ccn122.mcs.anl.gov -ccn123.mcs.anl.gov -ccn124.mcs.anl.gov -ccn125.mcs.anl.gov -ccn126.mcs.anl.gov -ccn127.mcs.anl.gov -ccn128.mcs.anl.gov -ccn129.mcs.anl.gov -ccn13.mcs.anl.gov -ccn130.mcs.anl.gov -ccn131.mcs.anl.gov -ccn132.mcs.anl.gov -ccn133.mcs.anl.gov -ccn134.mcs.anl.gov -ccn135.mcs.anl.gov -ccn136.mcs.anl.gov -ccn137.mcs.anl.gov -ccn138.mcs.anl.gov -ccn139.mcs.anl.gov -ccn14.mcs.anl.gov -ccn140.mcs.anl.gov -ccn141.mcs.anl.gov -ccn142.mcs.anl.gov -ccn143.mcs.anl.gov -ccn144.mcs.anl.gov -ccn145.mcs.anl.gov -ccn146.mcs.anl.gov -ccn147.mcs.anl.gov -ccn148.mcs.anl.gov -ccn149.mcs.anl.gov -ccn15.mcs.anl.gov -ccn150.mcs.anl.gov -ccn151.mcs.anl.gov -ccn152.mcs.anl.gov -ccn153.mcs.anl.gov -ccn154.mcs.anl.gov -ccn155.mcs.anl.gov -ccn156.mcs.anl.gov -ccn157.mcs.anl.gov -ccn158.mcs.anl.gov -ccn159.mcs.anl.gov -ccn16.mcs.anl.gov -ccn160.mcs.anl.gov -ccn161.mcs.anl.gov -ccn162.mcs.anl.gov -ccn163.mcs.anl.gov -ccn164.mcs.anl.gov -ccn165.mcs.anl.gov -ccn166.mcs.anl.gov -ccn167.mcs.anl.gov -ccn168.mcs.anl.gov -ccn169.mcs.anl.gov -ccn17.mcs.anl.gov -ccn170.mcs.anl.gov -ccn171.mcs.anl.gov -ccn172.mcs.anl.gov -ccn173.mcs.anl.gov -ccn174.mcs.anl.gov -ccn175.mcs.anl.gov -ccn176.mcs.anl.gov -ccn177.mcs.anl.gov -ccn178.mcs.anl.gov -ccn179.mcs.anl.gov -ccn18.mcs.anl.gov -ccn180.mcs.anl.gov -ccn181.mcs.anl.gov -ccn182.mcs.anl.gov -ccn183.mcs.anl.gov -ccn184.mcs.anl.gov -ccn185.mcs.anl.gov -ccn186.mcs.anl.gov -ccn187.mcs.anl.gov -ccn188.mcs.anl.gov -ccn189.mcs.anl.gov -ccn19.mcs.anl.gov -ccn190.mcs.anl.gov -ccn191.mcs.anl.gov -ccn192.mcs.anl.gov -ccn193.mcs.anl.gov -ccn194.mcs.anl.gov -ccn195.mcs.anl.gov -ccn196.mcs.anl.gov -ccn197.mcs.anl.gov -ccn198.mcs.anl.gov -ccn199.mcs.anl.gov -ccn2.mcs.anl.gov -ccn20.mcs.anl.gov -ccn200.mcs.anl.gov -ccn201.mcs.anl.gov -ccn202.mcs.anl.gov -ccn203.mcs.anl.gov -ccn204.mcs.anl.gov -ccn205.mcs.anl.gov -ccn206.mcs.anl.gov -ccn207.mcs.anl.gov -ccn208.mcs.anl.gov -ccn209.mcs.anl.gov -ccn21.mcs.anl.gov -ccn210.mcs.anl.gov -ccn211.mcs.anl.gov -ccn212.mcs.anl.gov -ccn213.mcs.anl.gov -ccn214.mcs.anl.gov -ccn215.mcs.anl.gov -ccn216.mcs.anl.gov -ccn217.mcs.anl.gov -ccn218.mcs.anl.gov -ccn219.mcs.anl.gov -ccn22.mcs.anl.gov -ccn220.mcs.anl.gov -ccn221.mcs.anl.gov -ccn222.mcs.anl.gov -ccn223.mcs.anl.gov -ccn224.mcs.anl.gov -ccn225.mcs.anl.gov -ccn226.mcs.anl.gov -ccn227.mcs.anl.gov -ccn228.mcs.anl.gov -ccn229.mcs.anl.gov -ccn23.mcs.anl.gov -ccn230.mcs.anl.gov -ccn231.mcs.anl.gov -ccn232.mcs.anl.gov -ccn233.mcs.anl.gov -ccn234.mcs.anl.gov -ccn235.mcs.anl.gov -ccn236.mcs.anl.gov -ccn237.mcs.anl.gov -ccn238.mcs.anl.gov -ccn239.mcs.anl.gov -ccn24.mcs.anl.gov -ccn240.mcs.anl.gov -ccn241.mcs.anl.gov -ccn242.mcs.anl.gov -ccn243.mcs.anl.gov -ccn244.mcs.anl.gov -ccn245.mcs.anl.gov -ccn246.mcs.anl.gov -ccn247.mcs.anl.gov -ccn248.mcs.anl.gov -ccn249.mcs.anl.gov -ccn25.mcs.anl.gov -ccn250.mcs.anl.gov -ccn251.mcs.anl.gov -ccn252.mcs.anl.gov -ccn253.mcs.anl.gov -ccn254.mcs.anl.gov -ccn255.mcs.anl.gov -ccn256.mcs.anl.gov -ccn26.mcs.anl.gov -ccn27.mcs.anl.gov -ccn28.mcs.anl.gov -ccn29.mcs.anl.gov -ccn3.mcs.anl.gov -ccn30.mcs.anl.gov -ccn31.mcs.anl.gov -ccn32.mcs.anl.gov -ccn33.mcs.anl.gov -ccn34.mcs.anl.gov -ccn35.mcs.anl.gov -ccn36.mcs.anl.gov -ccn37.mcs.anl.gov -ccn38.mcs.anl.gov -ccn39.mcs.anl.gov -ccn4.mcs.anl.gov -ccn40.mcs.anl.gov -ccn41.mcs.anl.gov -ccn42.mcs.anl.gov -ccn43.mcs.anl.gov -ccn44.mcs.anl.gov -ccn45.mcs.anl.gov -ccn46.mcs.anl.gov -ccn47.mcs.anl.gov -ccn48.mcs.anl.gov -ccn49.mcs.anl.gov -ccn5.mcs.anl.gov -ccn50.mcs.anl.gov -ccn51.mcs.anl.gov -ccn52.mcs.anl.gov -ccn53.mcs.anl.gov -ccn54.mcs.anl.gov -ccn55.mcs.anl.gov -ccn56.mcs.anl.gov -ccn57.mcs.anl.gov -ccn58.mcs.anl.gov -ccn59.mcs.anl.gov -ccn6.mcs.anl.gov -ccn60.mcs.anl.gov -ccn61.mcs.anl.gov -ccn62.mcs.anl.gov -ccn63.mcs.anl.gov -ccn64.mcs.anl.gov -ccn65.mcs.anl.gov -ccn66.mcs.anl.gov -ccn67.mcs.anl.gov -ccn68.mcs.anl.gov -ccn69.mcs.anl.gov -ccn7.mcs.anl.gov -ccn70.mcs.anl.gov -ccn71.mcs.anl.gov -ccn72.mcs.anl.gov -ccn73.mcs.anl.gov -ccn74.mcs.anl.gov -ccn75.mcs.anl.gov -ccn76.mcs.anl.gov -ccn77.mcs.anl.gov -ccn78.mcs.anl.gov -ccn79.mcs.anl.gov -ccn8.mcs.anl.gov -ccn80.mcs.anl.gov -ccn81.mcs.anl.gov -ccn82.mcs.anl.gov -ccn83.mcs.anl.gov -ccn84.mcs.anl.gov -ccn85.mcs.anl.gov -ccn86.mcs.anl.gov -ccn87.mcs.anl.gov -ccn88.mcs.anl.gov -ccn89.mcs.anl.gov -ccn9.mcs.anl.gov -ccn90.mcs.anl.gov -ccn91.mcs.anl.gov -ccn92.mcs.anl.gov -ccn93.mcs.anl.gov -ccn94.mcs.anl.gov -ccn95.mcs.anl.gov -ccn96.mcs.anl.gov -ccn97.mcs.anl.gov -ccn98.mcs.anl.gov -ccn99.mcs.anl.gov -ccnet.mcs.anl.gov -ccprez.mcs.anl.gov -ccreports.mcs.anl.gov -ccsched.mcs.anl.gov -ccspine1.mcs.anl.gov -ccspine2.mcs.anl.gov -ccspine3.mcs.anl.gov -ccspine4.mcs.anl.gov -ccsto1.mcs.anl.gov -ccsto2.mcs.anl.gov -ccsto3.mcs.anl.gov -ccsto4.mcs.anl.gov -ccsto5.mcs.anl.gov -ccsto6.mcs.anl.gov -ccsto7.mcs.anl.gov -ccsto8.mcs.anl.gov -cct0p1.mcs.anl.gov -cct0p2.mcs.anl.gov -cct0p3.mcs.anl.gov -cct10m.mcs.anl.gov -cct10net.mcs.anl.gov -cct10p1.mcs.anl.gov -cct10p2.mcs.anl.gov -cct10p3.mcs.anl.gov -cct10p4.mcs.anl.gov -cct10p5.mcs.anl.gov -cct1m.mcs.anl.gov -cct1net.mcs.anl.gov -cct1p1.mcs.anl.gov -cct1p2.mcs.anl.gov -cct1p3.mcs.anl.gov -cct1p4.mcs.anl.gov -cct1p5.mcs.anl.gov -cct2m.mcs.anl.gov -cct2net.mcs.anl.gov -cct2p1.mcs.anl.gov -cct2p2.mcs.anl.gov -cct2p3.mcs.anl.gov -cct2p4.mcs.anl.gov -cct2p5.mcs.anl.gov -cct3m.mcs.anl.gov -cct3net.mcs.anl.gov -cct3p1.mcs.anl.gov -cct3p2.mcs.anl.gov -cct3p3.mcs.anl.gov -cct3p4.mcs.anl.gov -cct3p5.mcs.anl.gov -cct4m.mcs.anl.gov -cct4net.mcs.anl.gov -cct4p1.mcs.anl.gov -cct4p2.mcs.anl.gov -cct4p3.mcs.anl.gov -cct4p4.mcs.anl.gov -cct4p5.mcs.anl.gov -cct5m.mcs.anl.gov -cct5net.mcs.anl.gov -cct5p1.mcs.anl.gov -cct5p2.mcs.anl.gov -cct5p3.mcs.anl.gov -cct5p4.mcs.anl.gov -cct5p5.mcs.anl.gov -cct6m.mcs.anl.gov -cct6net.mcs.anl.gov -cct6p1.mcs.anl.gov -cct6p2.mcs.anl.gov -cct6p3.mcs.anl.gov -cct6p4.mcs.anl.gov -cct6p5.mcs.anl.gov -cct7m.mcs.anl.gov -cct7net.mcs.anl.gov -cct7p1.mcs.anl.gov -cct7p2.mcs.anl.gov -cct7p3.mcs.anl.gov -cct7p4.mcs.anl.gov -cct7p5.mcs.anl.gov -cct8m.mcs.anl.gov -cct8net.mcs.anl.gov -cct8p1.mcs.anl.gov -cct8p2.mcs.anl.gov -cct8p3.mcs.anl.gov -cct8p4.mcs.anl.gov -cct8p5.mcs.anl.gov -cct9m.mcs.anl.gov -cct9net.mcs.anl.gov -cct9p1.mcs.anl.gov -cct9p2.mcs.anl.gov -cct9p3.mcs.anl.gov -cct9p4.mcs.anl.gov -cct9p5.mcs.anl.gov -cctest1.mcs.anl.gov -cctest2.mcs.anl.gov -cerebus.mcs.anl.gov -chalant.mcs.anl.gov -challmac.mcs.anl-external.org -champux.mcs.anl.gov -chapman.mcs.anl.gov -chastity-cph.mcs.anl.gov -checker.mcs.anl.gov -cheer-cph.mcs.anl.gov -chetiri.mcs.anl.gov -chevelle.mcs.anl.gov -chibalog.mcs.anl.gov -chimes-cph.mcs.anl.gov -choate.mcs.anl.gov -chomp.mcs.anl.gov -choochoo.mcs.anl.gov -ci-kdc-test.mcs.anl.gov -cicero.mcs.anl.gov -cipher.mcs.anl.gov -cis221-a216.mcs.anl.gov -cis221-a252.mcs.anl.gov -cis221-bmr-p22-1.mcs.anl.gov -cis221-bmr-p22-2.mcs.anl.gov -cis221-bmr-p22-3.mcs.anl.gov -cis221-d118.mcs.anl.gov -cl222-pos1-cph.mcs.anl.gov -clarinet.mcs.anl.gov -cliff.mcs.anl.gov -clone.mcs.anl.gov -cognito.mcs.anl.gov -cohoonsmac.mcs.anl.gov -contra.mcs.anl.gov -cooper.mcs.anl.gov -copper.mcs.anl.gov -corby.mcs.anl.gov -corelli.mcs.anl.gov -cosmo.mcs.anl.gov -couperin.mcs.anl.gov -couth.mcs.anl.gov -cphlinuxtest.mcs.anl.gov -cpmac.mcs.anl.gov -crestron.mcs.anl.gov -croxley.mcs.anl.gov -crunch.mcs.anl.gov -crushinator.mcs.anl.gov -crusoe.mcs.anl.gov -csi0332383.mcs.anl.gov -csi0332384.mcs.anl.gov -csi323012.mcs.anl.gov -csi325480.mcs.anl.gov -csi325706.mcs.anl.gov -csi327165.mcs.anl.gov -csi327168.mcs.anl.gov -csi327204.mcs.anl.gov -csi328919.mcs.anl.gov -csposcanner.for.testing.only -curfman.mcs.anl.gov -cutie.mcs.anl.gov -cvs.cogkit.org -d146-extreme.mcs.anl.gov -dark.mcs.anl.gov -darth.mcs.anl.gov -data0.i2u2.org -data1.i2u2.org -davinci.mcs.anl.gov -dawn-cph.mcs.anl.gov -daytripper.mcs.anl-external.org -dbs2.mcs.anl.gov -dbs3.mcs.anl.gov -ddn.kbt.mcs.anl.gov -ddnc1.kbt.mcs.anl.gov -ddnc2.kbt.mcs.anl.gov -death-cph.mcs.anl.gov -debiansummer03.mcs.anl.gov -delight-3-06p4-cph.mcs.anl.gov -delight.mcs.anl.gov -delta.mcs.anl.gov -deng.mcs.anl.gov -dengamd64.mcs.anl.gov -denkov.mcs.anl.gov -detert.mcs.anl.gov -dev.globus.org -digory.mcs.anl.gov -dilbert.mcs.anl.gov -diligence-cph.mcs.anl.gov -dimsum.mcs.anl.gov -dirac.mcs.anl.gov -distribs.mcs.anl.gov -dodongo.mcs.anl.gov -dogbert.mcs.anl.gov -donau.mcs.anl.gov -donner.mcs.anl.gov -double.mcs.anl.gov -doubt-cph.mcs.anl.gov -draft.mcs.anl.gov -dreamcast.mcs.anl.gov -drug-dev.nmpdr.org -drugan-desktop.mcs.anl.gov -ds4100-2a.nmpdr.org -ds4100-2b.nmpdr.org -dsl-a30p.mcs.anl.gov -dsl-csi324665.mcs.anl.gov -dsl-csi324666.mcs.anl.gov -dsl-csi326094.mcs.anl.gov -dsl-spare-a20p.mcs.anl.gov -dsl-spare-a22p.mcs.anl.gov -dsl-spare-a22p2.mcs.anl.gov -dsl-spare-a22p3.mcs.anl.gov -dsl-spare-t22-2.mcs.anl.gov -dsl-spare-t22-3.mcs.anl.gov -dsl-spare-t22.mcs.anl.gov -dsl-spare-t23-3.mcs.anl.gov -dsl-spare-t23.mcs.anl.gov -dsl-spare-t30.mcs.anl.gov -dsl-t30-spare.mcs.anl.gov -duluth.mcs.anl.gov -dunstable.mcs.anl.gov -dunwich.mcs.anl.gov -durante.mcs.anl.gov -durga.mcs.anl.gov -duscha.mcs.anl.gov -dusk-cph.mcs.anl.gov -dust2.mcs.anl.gov -dve.mcs.anl.gov -ecgtheow.mcs.anl.gov -echo.mcs.anl.gov -edno.mcs.anl.gov -effable.mcs.anl.gov -ejmac.mcs.anl.gov -elab.i2u2.org -elephant.mcs.anl.gov -elric.mcs.anl.gov -enigma.mcs.anl.gov -eolson-laptop.mcs.anl.gov -ephemere.mcs.anl.gov -eros.mcs.anl.gov -esleen.mcs.anl.gov -espresso.mcs.anl.gov -et212-241-154-cph.mcs.anl.gov -et212pc159-cph.mcs.anl.gov -ete.mcs.anl.gov -eternity-cph.mcs.anl.gov -europa.mcs.anl.gov -evaporated.mcs.anl.gov -eventum.mcs.anl.gov -evil.mcs.anl.gov -ext203-c229.mcs.anl.gov -ext221-a142.mcs.anl.gov -ext221-bmr-ad47.mcs.anl.gov -ext221-c139.mcs.anl.gov -ext221-d146.mcs.anl.gov -ext221-d255-1.mcs.anl.gov -ext221-d255-2.mcs.anl.gov -falco.mcs.anl.gov -faultytower.mcs.anl.gov -fc-2.nmpdr.org -fdr.mcs.anl.gov -feegle.mcs.anl.gov -feigl-laptop.mcs.anl.gov -feynman.mcs.anl.gov -finster.mcs.anl.gov -fireintheh0le.org -fischer-laptop.mcs.anl.gov -fischer-t40p.mcs.anl.gov -fission.mcs.anl.gov -fl-athlon-64.mcs.anl.gov -fl-caserver.mcs.anl.gov -fl-cvs.mcs.anl.gov -fl-gentoo-build.mcs.anl.gov -fl-login.mcs.anl.gov -fl-viewpad.mcs.anl.gov -fl-windows.mcs.anl.gov -flake.mcs.anl.gov -flea.mcs.anl-external.org -float20015.mcs.anl-external.org -float20016.mcs.anl-external.org -float20017.mcs.anl-external.org -float20018.mcs.anl-external.org -float20019.mcs.anl-external.org -float20020.mcs.anl-external.org -float20021.mcs.anl-external.org -float20022.mcs.anl-external.org -float20023.mcs.anl-external.org -float20024.mcs.anl-external.org -float20025.mcs.anl-external.org -float20026.mcs.anl-external.org -float20027.mcs.anl-external.org -float20028.mcs.anl-external.org -float20036.mcs.anl-external.org -float20037.mcs.anl-external.org -float20038.mcs.anl-external.org -float20039.mcs.anl-external.org -float20040.mcs.anl-external.org -float340121.mcs.anl-external.org -float340122.mcs.anl-external.org -float340123.mcs.anl-external.org -float34064.mcs.anl-external.org -float34082.mcs.anl-external.org -float34092.mcs.anl-external.org -float34093.mcs.anl-external.org -float34094.mcs.anl-external.org -float34124.mcs.anl-external.org -float58218.mcs.anl-external.org -float58219.mcs.anl-external.org -folkers.mcs.anl.gov -forge.teragrid.org -fortitude-cph.mcs.anl.gov -foster-mac.mcs.anl.gov -foster-t60-2.mcs.anl.gov -foster-t60.mcs.anl.gov -foster.mcs.anl.gov -foxtrot.mcs.anl.gov -fpga01.mcs.anl.gov -fpga02.mcs.anl.gov -fpga03.mcs.anl.gov -fpga04.mcs.anl.gov -frame.mcs.anl.gov -fraser.mcs.anl.gov -freaky.mcs.anl.gov -frenchhorn.mcs.anl.gov -freschl.mcs.anl.gov -frink.mcs.anl.gov -fs1.bgl.mcs.anl.gov -fs10.bgl.mcs.anl.gov -fs11.bgl.mcs.anl.gov -fs12.bgl.mcs.anl.gov -fs13.bgl.mcs.anl.gov -fs14.bgl.mcs.anl.gov -fs15.bgl.mcs.anl.gov -fs16.bgl.mcs.anl.gov -fs2.bgl.mcs.anl.gov -fs3.bgl.mcs.anl.gov -fs4.bgl.mcs.anl.gov -fs5.bgl.mcs.anl.gov -fs6.bgl.mcs.anl.gov -fs7.bgl.mcs.anl.gov -fs8.bgl.mcs.anl.gov -fs9.bgl.mcs.anl.gov -ftp.mpi-forum.org -ftp.nsf-middleware.org -fujita.mcs.anl.gov -fun-cph.mcs.anl.gov -fusion.anl.gov -fuzzy.mcs.anl.gov -fwzswbcv786.net.anl.gov -fwzswgwv786.net.anl.gov -fwzswidv786.net.anl.gov -gadu.mcs.anl.gov -gainly.mcs.anl.gov -gallium.mcs.anl.gov -gaspode.mcs.anl.gov -gce.cogkit.org -geminiani.mcs.anl.gov -genesdesk.for.testing.only -geneslinux.for.testing.only -georgia.mcs.anl.gov -geowall.mcs.anl.gov -gibbons.mcs.anl.gov -gibraltar.mcs.anl.gov -gigi.mcs.anl.gov -gigi2.mcs.anl.gov -gigi3.mcs.anl.gov -gil.mcs.anl.gov -girard.mcs.anl.gov -gizah.mcs.anl.gov -gizmodgery.mcs.anl.gov -glob.mcs.anl.gov -globuscvs.mcs.anl-external.org -gnare-dev.mcs.anl.gov -gnare.mcs.anl.gov -gollyg.mcs.anl.gov -goltsman.mcs.anl.gov -gong.mcs.anl.gov -goomba.mcs.anl.gov -gorilla.mcs.anl.gov -gose.mcs.anl.gov -gossamer.mcs.anl.gov -grace-cph.mcs.anl.gov -gravitino.mcs.anl.gov -gravitude.mcs.anl.gov -gregurich-t20.mcs.anl.gov -gridftp.mcs.anl.gov -gridshib.globus.org -griffin-r51.mcs.anl.gov -gropp-t42.mcs.anl.gov -groppmac.mcs.anl.gov -gruntled.mcs.anl.gov -guitar.mcs.anl.gov -gumby.mcs.anl.gov -gun.mcs.anl.gov -guss.mcs.anl.gov -gust.mcs.anl.gov -gusto-cph.mcs.anl.gov -gw.mcs.anl.gov -handel.mcs.anl.gov -handshake.mcs.anl.gov -harley.mcs.anl.gov -harmony-cph.mcs.anl.gov -hatshepsut.mcs.anl.gov -haydn.mcs.anl.gov -hd-600-loaner.mcs.anl.gov -hd-loaner-x21.mcs.anl.gov -heinzel.mcs.anl.gov -helium.mcs.anl.gov -helix.mcs.anl.gov -hermit.mcs.anl.gov -hiddencat.mcs.anl.gov -hmc1.bgl.mcs.anl.gov -hobbes.mcs.anl.gov -hook.mcs.anl.gov -hookshot.mcs.anl.gov -horikawa-cph.mcs.anl.gov -hostbase.mcs.anl.gov -hovland.mcs.anl.gov -hpc-221sw-lo0.mcs.anl.gov -hpss.mcs.anl.gov -hs1.bgl.mcs.anl.gov -hs2.bgl.mcs.anl.gov -hs3.bgl.mcs.anl.gov -hs4.bgl.mcs.anl.gov -hsrp.mcs.anl.gov -huma.mcs.anl.gov -hume.mcs.anl.gov -humility-cph.mcs.anl.gov -hunterofcow.mcs.anl.gov -hurdygurdy.mcs.anl.gov -ianmobile.mcs.anl.gov -ibook-g4.mcs.anl.gov -ice.mcs.anl.gov -icicle.mcs.anl.gov -icrunch.mcs.anl.gov -ilap.mcs.anl.gov -ilsig.mcs.anl.gov -imacat.mcs.anl.gov -imaclam.mcs.anl.gov -incapable.mcs.anl-external.org -insepov-cph.mcs.anl.gov -inspire.mcs.anl.gov -intel1.mcs.anl.gov -ira.mcs.anl-external.org -irnbru.mcs.anl.gov -irving.mcs.anl.gov -j1.lcrc.anl.gov -j10.lcrc.anl.gov -j100.lcrc.anl.gov -j101.lcrc.anl.gov -j102.lcrc.anl.gov -j103.lcrc.anl.gov -j104.lcrc.anl.gov -j105.lcrc.anl.gov -j106.lcrc.anl.gov -j107.lcrc.anl.gov -j108.lcrc.anl.gov -j109.lcrc.anl.gov -j11.lcrc.anl.gov -j110.lcrc.anl.gov -j111.lcrc.anl.gov -j112.lcrc.anl.gov -j113.lcrc.anl.gov -j114.lcrc.anl.gov -j115.lcrc.anl.gov -j116.lcrc.anl.gov -j117.lcrc.anl.gov -j118.lcrc.anl.gov -j119.lcrc.anl.gov -j12.lcrc.anl.gov -j120.lcrc.anl.gov -j121.lcrc.anl.gov -j122.lcrc.anl.gov -j123.lcrc.anl.gov -j124.lcrc.anl.gov -j125.lcrc.anl.gov -j126.lcrc.anl.gov -j127.lcrc.anl.gov -j128.lcrc.anl.gov -j129.lcrc.anl.gov -j13.lcrc.anl.gov -j130.lcrc.anl.gov -j131.lcrc.anl.gov -j132.lcrc.anl.gov -j133.lcrc.anl.gov -j134.lcrc.anl.gov -j135.lcrc.anl.gov -j136.lcrc.anl.gov -j137.lcrc.anl.gov -j138.lcrc.anl.gov -j139.lcrc.anl.gov -j14.lcrc.anl.gov -j140.lcrc.anl.gov -j141.lcrc.anl.gov -j142.lcrc.anl.gov -j143.lcrc.anl.gov -j144.lcrc.anl.gov -j145.lcrc.anl.gov -j146.lcrc.anl.gov -j147.lcrc.anl.gov -j148.lcrc.anl.gov -j149.lcrc.anl.gov -j15.lcrc.anl.gov -j150.lcrc.anl.gov -j151.lcrc.anl.gov -j152.lcrc.anl.gov -j153.lcrc.anl.gov -j154.lcrc.anl.gov -j155.lcrc.anl.gov -j156.lcrc.anl.gov -j157.lcrc.anl.gov -j158.lcrc.anl.gov -j159.lcrc.anl.gov -j16.lcrc.anl.gov