diff options
Diffstat (limited to 'tensorflow/contrib/verbs/rdma.cc')
| -rw-r--r-- | tensorflow/contrib/verbs/rdma.cc | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/tensorflow/contrib/verbs/rdma.cc b/tensorflow/contrib/verbs/rdma.cc index 53d840f5d1..05df05de35 100644 --- a/tensorflow/contrib/verbs/rdma.cc +++ b/tensorflow/contrib/verbs/rdma.cc @@ -271,6 +271,11 @@ RdmaChannel::RdmaChannel(const RdmaAdapter* adapter, const string local_name, self_.lid = attr.lid; self_.qpn = qp_->qp_num; self_.psn = static_cast<uint32_t>(random::New64()) & 0xffffff; + union ibv_gid gid; + CHECK(!ibv_query_gid(adapter_->context_, (uint8_t)1, 0, &gid)) + << "Query gid"; + self_.snp = gid.global.subnet_prefix; + self_.iid = gid.global.interface_id; } // create message and ack buffers, then initialize the tables. @@ -320,11 +325,15 @@ void RdmaChannel::SetRemoteAddress(const RdmaAddress& ra, bool override) { remote_.lid = ra.lid; remote_.qpn = ra.qpn; remote_.psn = ra.psn; + remote_.snp = ra.snp; + remote_.iid = ra.iid; remote_set_ = true; } else { CHECK(remote_.lid == ra.lid); CHECK(remote_.qpn == ra.qpn); CHECK(remote_.psn == ra.psn); + CHECK(remote_.snp == ra.snp); + CHECK(remote_.iid == ra.iid); } } @@ -472,7 +481,11 @@ void RdmaChannel::Connect(const RdmaAddress& remoteAddr) { attr.rq_psn = remoteAddr.psn; attr.max_dest_rd_atomic = 1; attr.min_rnr_timer = 12; - attr.ah_attr.is_global = 0; + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.dgid.global.subnet_prefix = remoteAddr.snp; + attr.ah_attr.grh.dgid.global.interface_id = remoteAddr.iid; + attr.ah_attr.grh.flow_label = 0; + attr.ah_attr.grh.hop_limit = 255; attr.ah_attr.dlid = remoteAddr.lid; attr.ah_attr.sl = 0; attr.ah_attr.src_path_bits = 0; |