aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/contrib/verbs/rdma.h
diff options
context:
space:
mode:
Diffstat (limited to 'tensorflow/contrib/verbs/rdma.h')
-rw-r--r--tensorflow/contrib/verbs/rdma.h40
1 files changed, 35 insertions, 5 deletions
diff --git a/tensorflow/contrib/verbs/rdma.h b/tensorflow/contrib/verbs/rdma.h
index e1e07db776..52d92a7c5b 100644
--- a/tensorflow/contrib/verbs/rdma.h
+++ b/tensorflow/contrib/verbs/rdma.h
@@ -36,7 +36,24 @@ limitations under the License.
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
-
+#define PKEY_DEFAULT 0
+#define QUEUE_DEPTH_DEFAULT 1024
+#define TIMEOUT_DEFAULT 14
+#define RETRY_CNT_DEFAULT 7
+#define SL_DEFAULT 0
+#define TRAFFIC_CLASS 0
+
+struct RdmaParams {
+ uint8_t port_num;
+ uint8_t sgid_index;
+ uint8_t pkey_index;
+ uint32_t queue_depth;
+ uint8_t timeout;
+ uint8_t retry_cnt;
+ uint8_t sl;
+ enum ibv_mtu mtu;
+ uint8_t traffic_class;
+};
// structure to save the address of remote channels.
struct RdmaAddress {
uint32_t lid;
@@ -50,9 +67,20 @@ struct RemoteMR {
uint64_t remote_addr;
uint32_t rkey;
};
-enum BufferStatus { none, idle, busy };
-enum Location { local, remote };
-enum BufferType { ACK, MESSAGE, TENSOR };
+enum BufferStatus {
+ none,
+ idle,
+ busy
+};
+enum Location {
+ local,
+ remote
+};
+enum BufferType {
+ ACK,
+ MESSAGE,
+ TENSOR
+};
enum RdmaMessageType {
RDMA_MESSAGE_ACK,
RDMA_MESSAGE_BUFFER_IDLE,
@@ -84,6 +112,8 @@ class RdmaAdapter {
protected:
static const int MAX_CONCURRENT_WRITES = 1000;
ibv_context* context_;
+ // RDMA configuration parameters
+ RdmaParams params_;
// ibverbs protection domain
ibv_pd* pd_;
// Completion event channel, to wait for work completions
@@ -183,7 +213,7 @@ class RdmaBuffer {
}
void FreeBuffer();
void EnqueueItem(string Item);
- virtual void SendNextItem(){};
+ virtual void SendNextItem() {};
void CreateCPUBuffer(size_t size, bool lock = true);
void SetRemoteMR(RemoteMR rmi, bool override);
uint32_t LookupBufferIndex(const string& buffer_name) {