From ce78eb62a3684355829e159943235fc2c045b3ca Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 4 Aug 2021 01:20:19 -0400 Subject: [PATCH 01/39] Added --with-spectrum-mpi to configure --- Makefile.am | 6 +++--- configure.ac | 44 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index a66dfda..ac9a0cc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -36,7 +36,7 @@ bin_PROGRAMS = tests/gds_kernel_latency tests/gds_poll_lat tests/gds_kernel_loop noinst_PROGRAMS = tests/rstest tests/wqtest tests_gds_kernel_latency_SOURCES = tests/gds_kernel_latency.c tests/gpu_kernels.cu tests/pingpong.c tests/gpu.cpp -tests_gds_kernel_latency_LDADD = $(top_builddir)/src/libgdsync.la -lmpi $(LIBGDSTOOLS) -lgdrapi $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) +tests_gds_kernel_latency_LDADD = $(top_builddir)/src/libgdsync.la $(MPILDFLAGS) $(LIBGDSTOOLS) -lgdrapi $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) tests_rstest_SOURCES = tests/rstest.cpp tests_rstest_LDADD = @@ -45,10 +45,10 @@ tests_wqtest_SOURCES = tests/task_queue_test.cpp tests_wqtest_LDADD = $(PTHREAD_LIBS) tests_gds_poll_lat_SOURCES = tests/gds_poll_lat.c tests/gpu.cpp tests/gpu_kernels.cu -tests_gds_poll_lat_LDADD = $(top_builddir)/src/libgdsync.la $(LIBGDSTOOLS) -lgdrapi -lmpi $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) +tests_gds_poll_lat_LDADD = $(top_builddir)/src/libgdsync.la $(LIBGDSTOOLS) -lgdrapi $(MPILDFLAGS) $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) tests_gds_sanity_SOURCES = tests/gds_sanity.cpp tests/gpu.cpp tests/gpu_kernels.cu -tests_gds_sanity_LDADD = $(top_builddir)/src/libgdsync.la $(LIBGDSTOOLS) -lgdrapi -lmpi $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) +tests_gds_sanity_LDADD = $(top_builddir)/src/libgdsync.la $(LIBGDSTOOLS) -lgdrapi $(MPILDFLAGS) $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) tests_gds_kernel_loopback_latency_SOURCES = tests/gds_kernel_loopback_latency.c tests/pingpong.c tests/gpu.cpp tests/gpu_kernels.cu tests_gds_kernel_loopback_latency_LDADD = $(top_builddir)/src/libgdsync.la $(LIBGDSTOOLS) -lgdrapi $(LIBNVTX) -lcuda -lcudart $(PTHREAD_LIBS) diff --git a/configure.ac b/configure.ac index a79aed6..e20f313 100644 --- a/configure.ac +++ b/configure.ac @@ -93,25 +93,54 @@ else AC_SUBST(LIBGDSTOOLS) fi -AC_ARG_WITH([mpi], - AC_HELP_STRING([--with-mpi], [ Set path to mpi installation ])) -if test x$with_mpi = x || test x$with_mpi = xno; then +AC_ARG_WITH([spectrum-mpi], + AC_HELP_STRING([--with-spectrum-mpi], [ Set path to Spectrum MPI installation ])) +if test x$with_spectrum_mpi = x || test x$with_spectrum_mpi = xno; then # assuming system location mpi_home=/usr - MPICC=$with_home/bin/mpicc - MPICXX=$with_home/bin/mpic++ + MPICC=/bin/mpicc + MPICXX=/bin/mpic++ + MPILDFLAGS="-lmpi_ibm" else - if test -d $with_mpi; then - mpi_home=$with_mpi + if test -d $with_spectrum_mpi; then + mpi_home=$with_spectrum_mpi MPICC=${mpi_home}/bin/mpicc MPICXX=${mpi_home}/bin/mpic++ CPPFLAGS="$CPPFLAGS -I${mpi_home}/include" LDFLAGS="$LDFLAGS -L${mpi_home}/lib -L${mpi_home}/lib64" + MPILDFLAGS="-lmpi_ibm" else echo "MPI dir does not exist" fi fi +AC_ARG_WITH([mpi], + AC_HELP_STRING([--with-mpi], [ Set path to MPI installation ])) +if test x$with_spectrum_mpi = x || test x$with_spectrum_mpi == xno; then + if test x$with_mpi = x || test x$with_mpi = xno; then + # assuming system location + mpi_home=/usr + MPICC=/bin/mpicc + MPICXX=/bin/mpic++ + MPILDFLAGS="-lmpi" + else + if test -d $with_mpi; then + mpi_home=$with_mpi + MPICC=${mpi_home}/bin/mpicc + MPICXX=${mpi_home}/bin/mpic++ + CPPFLAGS="$CPPFLAGS -I${mpi_home}/include" + LDFLAGS="$LDFLAGS -L${mpi_home}/lib -L${mpi_home}/lib64" + MPILDFLAGS="-lmpi" + else + echo "MPI dir does not exist" + fi + fi +fi + +if test x$with_spectrum_mpi != x && test x$with_spectrum_mpi != xno && test x$with_mpi != x && test x$with_mpi != xno; then + AC_MSG_ERROR([--with-mpi and --with-spectrum-mpi are mutually exclusive.]) +fi + dnl Specify CUDA Location AC_ARG_WITH(cuda-toolkit, AC_HELP_STRING([--with-cuda-toolkit=CUDATKDIR], [ Specify CUDA toolkit installation directory (default: /usr/local/cuda)]), @@ -186,6 +215,7 @@ AC_MSG_NOTICE([Setting MPI_PATH = ${mpi_home} ]) AC_SUBST( MPI_PATH, [${mpi_home} ]) AC_SUBST( MPICC, [${MPICC} ]) AC_SUBST( MPICXX, [${MPICXX} ]) +AC_SUBST( MPILDFLAGS, [${MPILDFLAGS} ]) CPPFLAGS="$CPPFLAGS -I$CUDA_DRV_PATH/include -I$CUDA_PATH/include" LDFLAGS="$LDFLAGS -L$CUDA_DRV_PATH/lib64 -L$CUDA_DRV_PATH/lib -L$CUDA_PATH/lib64 -L$CUDA_PATH/lib" From 66409ac64785c884889e3d85df10a3a34bc2e8f9 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 4 Aug 2021 21:42:55 -0400 Subject: [PATCH 02/39] Introduced GDS_DRIVER_TYPE to gds_qp and gds_cq --- include/gdsync/core.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index 7ff0cbb..87a4cc3 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -40,26 +40,33 @@ ((((v) & 0x0000ffffU) >> 0 ) >= (unsigned)GDS_API_MINOR_VERSION) ) typedef enum gds_param { - GDS_PARAM_VERSION, - GDS_NUM_PARAMS + GDS_PARAM_VERSION, + GDS_NUM_PARAMS } gds_param_t; int gds_query_param(gds_param_t param, int *value); enum gds_create_qp_flags { - GDS_CREATE_QP_DEFAULT = 0, - GDS_CREATE_QP_WQ_ON_GPU = 1<<0, - GDS_CREATE_QP_TX_CQ_ON_GPU = 1<<1, - GDS_CREATE_QP_RX_CQ_ON_GPU = 1<<2, - GDS_CREATE_QP_WQ_DBREC_ON_GPU = 1<<5, + GDS_CREATE_QP_DEFAULT = 0, + GDS_CREATE_QP_WQ_ON_GPU = 1<<0, + GDS_CREATE_QP_TX_CQ_ON_GPU = 1<<1, + GDS_CREATE_QP_RX_CQ_ON_GPU = 1<<2, + GDS_CREATE_QP_WQ_DBREC_ON_GPU = 1<<5, }; typedef struct ibv_exp_qp_init_attr gds_qp_init_attr_t; typedef struct ibv_exp_send_wr gds_send_wr; +typedef enum gds_driver_type { + GDS_DRIVER_TYPE_MLX5_EXP = 0, + GDS_DRIVER_TYPE_MLX5_DV, + GDS_DRIVER_TYPE_MLX5_DEVX +} gds_driver_type_t; + struct gds_cq { struct ibv_cq *cq; uint32_t curr_offset; + gds_driver_type_t dtype; }; struct gds_qp { @@ -68,6 +75,7 @@ struct gds_qp { struct gds_cq recv_cq; struct ibv_exp_res_domain * res_domain; struct ibv_context *dev_context; + gds_driver_type_t dtype; }; /* \brief: Create a peer-enabled QP attached to the specified GPU id. From fdd2270a1a2736657c6385e75d0991b6146a6829 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 4 Aug 2021 21:43:15 -0400 Subject: [PATCH 03/39] Set gds_qp and gds_cq dtype to MLX5_EXP in the creation functions --- src/gdsync.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 90d5508..e863ec0 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1754,6 +1754,8 @@ gds_create_cq_internal(struct ibv_context *context, int cqe, return NULL; } + gcq->dtype = GDS_DRIVER_TYPE_MLX5_EXP; + return gcq; } @@ -1893,6 +1895,7 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gqp->send_cq.curr_offset = 0; gqp->recv_cq.cq = qp->recv_cq; gqp->recv_cq.curr_offset = 0; + gqp->dtype = GDS_DRIVER_TYPE_MLX5_EXP; gds_dbg("created gds_qp=%p\n", gqp); From 0073f813b84bf27579ba5d6868808fc349baef4d Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 4 Aug 2021 22:34:26 -0400 Subject: [PATCH 04/39] Changed the definition of gds_qp_init_attr_t and gds_send_wr. Also, removed res_domain from the gds_qp struct --- include/gdsync/core.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index 87a4cc3..4b61e86 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -54,8 +54,8 @@ enum gds_create_qp_flags { GDS_CREATE_QP_WQ_DBREC_ON_GPU = 1<<5, }; -typedef struct ibv_exp_qp_init_attr gds_qp_init_attr_t; -typedef struct ibv_exp_send_wr gds_send_wr; +typedef struct ibv_qp_init_attr gds_qp_init_attr_t; +typedef struct ibv_send_wr gds_send_wr; typedef enum gds_driver_type { GDS_DRIVER_TYPE_MLX5_EXP = 0, @@ -73,7 +73,6 @@ struct gds_qp { struct ibv_qp *qp; struct gds_cq send_cq; struct gds_cq recv_cq; - struct ibv_exp_res_domain * res_domain; struct ibv_context *dev_context; gds_driver_type_t dtype; }; From 78eadbb4c899d66c8073f4e8e26ed79ed6a15d9a Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 01:26:25 -0400 Subject: [PATCH 05/39] Implemented gds_get_driver_type --- include/gdsync/core.h | 7 ++++--- src/utils.hpp | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index 4b61e86..c438594 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -58,7 +58,8 @@ typedef struct ibv_qp_init_attr gds_qp_init_attr_t; typedef struct ibv_send_wr gds_send_wr; typedef enum gds_driver_type { - GDS_DRIVER_TYPE_MLX5_EXP = 0, + GDS_DRIVER_TYPE_UNSUPPORTED = 0, + GDS_DRIVER_TYPE_MLX5_EXP, GDS_DRIVER_TYPE_MLX5_DV, GDS_DRIVER_TYPE_MLX5_DEVX } gds_driver_type_t; @@ -71,8 +72,8 @@ struct gds_cq { struct gds_qp { struct ibv_qp *qp; - struct gds_cq send_cq; - struct gds_cq recv_cq; + struct gds_cq *send_cq; + struct gds_cq *recv_cq; struct ibv_context *dev_context; gds_driver_type_t dtype; }; diff --git a/src/utils.hpp b/src/utils.hpp index b501bda..de2aaa8 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -222,6 +222,22 @@ gds_peer *peer_from_stream(CUstream stream); //----------------------------------------------------------------------------- +/* \brief: Get the underlying driver associated with the ibdev. + * + */ +static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) +{ + const char *dev_name = ibv_get_device_name(ibdev); + + // Heuristically guess the driver by the device name. + // Until we find a better way to do so... + if (strstr(dev_name, "mlx5") != NULL) + return GDS_DRIVER_TYPE_MLX5_EXP; + return GDS_DRIVER_TYPE_UNSUPPORTED; +} + +//----------------------------------------------------------------------------- + /* * Local variables: * c-indent-level: 8 From aade2abf15285314c88841a9a3b7c8d0948f0afc Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 01:27:08 -0400 Subject: [PATCH 06/39] Initial implementation of mlx5-exp.cpp/hpp and moved create/destroy qp/cq functions to mlx5-exp --- src/gdsync.cpp | 125 ------------------------ src/mlx5-exp.cpp | 245 +++++++++++++++++++++++++++++++++++++++++++++++ src/mlx5-exp.hpp | 20 ++++ 3 files changed, 265 insertions(+), 125 deletions(-) create mode 100644 src/mlx5-exp.cpp create mode 100644 src/mlx5-exp.hpp diff --git a/src/gdsync.cpp b/src/gdsync.cpp index e863ec0..5a14c51 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1678,131 +1678,6 @@ gds_peer *peer_from_stream(CUstream stream) //----------------------------------------------------------------------------- -static ibv_exp_res_domain *gds_create_res_domain(struct ibv_context *context) -{ - if (!context) { - gds_err("invalid context"); - return NULL; - } - - ibv_exp_res_domain_init_attr res_domain_attr; - memset(&res_domain_attr, 0, sizeof(res_domain_attr)); - - res_domain_attr.comp_mask |= IBV_EXP_RES_DOMAIN_THREAD_MODEL; - res_domain_attr.thread_model = IBV_EXP_THREAD_SINGLE; - - ibv_exp_res_domain *res_domain = ibv_exp_create_res_domain(context, &res_domain_attr); - if (!res_domain) { - gds_warn("Can't create resource domain\n"); - } - - return res_domain; -} - -//----------------------------------------------------------------------------- - -static struct gds_cq * -gds_create_cq_internal(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, - struct ibv_exp_res_domain * res_domain) -{ - struct gds_cq *gcq = NULL; - ibv_exp_cq_init_attr attr; - gds_peer *peer = NULL; - gds_peer_attr *peer_attr = NULL; - int ret=0; - - if(!context) - { - gds_dbg("Invalid input context\n"); - return NULL; - } - - gcq = (struct gds_cq*)calloc(1, sizeof(struct gds_cq)); - if (!gcq) { - gds_err("cannot allocate memory\n"); - return NULL; - } - - //Here we need to recover peer and peer_attr pointers to set alloc_type and alloc_flags - //before ibv_exp_create_cq - ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); - if (ret) { - gds_err("error %d while registering GPU peer\n", ret); - return NULL; - } - assert(peer); - assert(peer_attr); - - peer->alloc_type = gds_peer::CQ; - peer->alloc_flags = flags; - - attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT; - attr.flags = 0; // see ibv_exp_cq_create_flags - attr.peer_direct_attrs = peer_attr; - if (res_domain) { - gds_dbg("using peer->res_domain %p for CQ\n", res_domain); - attr.res_domain = res_domain; - attr.comp_mask |= IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN; - } - - int old_errno = errno; - gcq->cq = ibv_exp_create_cq(context, cqe, cq_context, channel, comp_vector, &attr); - if (!gcq->cq) { - gds_err("error %d in ibv_exp_create_cq, old errno %d\n", errno, old_errno); - return NULL; - } - - gcq->dtype = GDS_DRIVER_TYPE_MLX5_EXP; - - return gcq; -} - -//Note: general create cq function, not really used for now! -struct gds_cq * -gds_create_cq(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags) -{ - int ret = 0; - struct gds_cq *gcq = NULL; - //TODO: leak of res_domain - struct ibv_exp_res_domain * res_domain; - gds_dbg("cqe=%d gpu_id=%d cq_flags=%08x\n", cqe, gpu_id, flags); - - gds_peer *peer = NULL; - gds_peer_attr *peer_attr = NULL; - ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); - if (ret) { - gds_err("error %d while registering GPU peer\n", ret); - return NULL; - } - assert(peer); - assert(peer_attr); - - peer->alloc_type = gds_peer::CQ; - peer->alloc_flags = flags; - - res_domain = gds_create_res_domain(context); - if (res_domain) - gds_dbg("using res_domain %p\n", res_domain); - else - gds_warn("NOT using res_domain\n"); - - - gcq = gds_create_cq_internal(context, cqe, cq_context, channel, comp_vector, gpu_id, flags, res_domain); - - if (!gcq) { - gds_err("error in gds_create_cq_internal\n"); - return NULL; - } - - return gcq; -} - -//----------------------------------------------------------------------------- - struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) { diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp new file mode 100644 index 0000000..9e4efe4 --- /dev/null +++ b/src/mlx5-exp.cpp @@ -0,0 +1,245 @@ +#include +#include +#include + +#include "mlx5-exp.hpp" +#include "utils.hpp" + +static ibv_exp_res_domain *gds_mlx5_exp_create_res_domain(struct ibv_context *context) +{ + if (!context) { + gds_err("invalid context"); + return NULL; + } + + ibv_exp_res_domain_init_attr res_domain_attr; + memset(&res_domain_attr, 0, sizeof(res_domain_attr)); + + res_domain_attr.comp_mask |= IBV_EXP_RES_DOMAIN_THREAD_MODEL; + res_domain_attr.thread_model = IBV_EXP_THREAD_SINGLE; + + ibv_exp_res_domain *res_domain = ibv_exp_create_res_domain(context, &res_domain_attr); + if (!res_domain) { + gds_warn("Can't create resource domain\n"); + } + + return res_domain; +} + +//----------------------------------------------------------------------------- + +static struct gds_mlx5_exp_cq_t * +gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, + struct ibv_exp_res_domain *res_domain) +{ + struct gds_mlx5_exp_cq_t *gmexpcq = NULL; + ibv_exp_cq_init_attr attr; + gds_peer *peer = NULL; + gds_peer_attr *peer_attr = NULL; + int ret = 0; + + if (!context) + { + gds_dbg("Invalid input context\n"); + return NULL; + } + + gmexpcq = (gds_mlx5_exp_cq_t *)calloc(1, sizeof(gds_mlx5_exp_cq_t)); + if (!gmexpcq) { + gds_err("cannot allocate memory\n"); + return NULL; + } + + //Here we need to recover peer and peer_attr pointers to set alloc_type and alloc_flags + //before ibv_exp_create_cq + ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); + if (ret) { + gds_err("error %d while registering GPU peer\n", ret); + return NULL; + } + assert(peer); + assert(peer_attr); + + peer->alloc_type = gds_peer::CQ; + peer->alloc_flags = flags; + + attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT; + attr.flags = 0; // see ibv_exp_cq_create_flags + attr.peer_direct_attrs = peer_attr; + if (res_domain) { + gds_dbg("using peer->res_domain %p for CQ\n", res_domain); + attr.res_domain = res_domain; + attr.comp_mask |= IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN; + gmexpcq->res_domain = res_domain; + } + + int old_errno = errno; + gmexpcq->gcq.cq = ibv_exp_create_cq(context, cqe, cq_context, channel, comp_vector, &attr); + if (!gmexpcq->gcq.cq) { + gds_err("error %d in ibv_exp_create_cq, old errno %d\n", errno, old_errno); + return NULL; + } + + gmexpcq->gcq.dtype = GDS_DRIVER_TYPE_MLX5_EXP; + + return gmexpcq; +} + +//----------------------------------------------------------------------------- + +struct gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, + gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) +{ + int ret = 0; + gds_mlx5_exp_qp_t *gmexpqp = NULL; + struct ibv_qp *qp = NULL; + gds_mlx5_exp_cq_t *rx_gmexpcq = NULL, *tx_gmexpcq = NULL; + gds_peer *peer = NULL; + gds_peer_attr *peer_attr = NULL; + struct ibv_qp_init_attr exp_qp_attr = {0,}; + int old_errno = errno; + + assert(pd); + assert(context); + assert(qp_attr); + + gmexpqp = (gds_mlx5_exp_qp_t *)calloc(1, sizeof(struct gds_qp)); + if (!gqp) { + gds_err("cannot allocate memory\n"); + return NULL; + } + gmexpqp->gqp.dtype = GDS_DRIVER_TYPE_MLX5_EXP; + + gmexpqp->gqp.dev_context = context; + + // peer registration + gds_dbg("before gds_register_peer_ex\n"); + ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); + if (ret) { + gds_err("error %d in gds_register_peer_ex\n", ret); + goto err; + } + + gmexpqp->res_domain = gds_create_res_domain(context); + if (gmexpqp->res_domain) + gds_dbg("using res_domain %p\n", gmexpqp->res_domain); + else + gds_warn("NOT using res_domain\n"); + + tx_gmexpcq = gds_mlx5_exp_create_cq(context, qp_attr->cap.max_send_wr, NULL, NULL, 0, gpu_id, + (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, + gmexpqp->res_domain); + if (!tx_gmexpcq) { + ret = errno; + gds_err("error %d while creating TX CQ, old_errno=%d\n", ret, old_errno); + goto err; + } + + rx_gmexpcq = gds_mlx5_exp_create_cq(context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, gpu_id, + (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, + gmexpqp->res_domain); + if (!rx_gmexpcq) { + ret = errno; + gds_err("error %d while creating RX CQ\n", ret); + goto err; + } + + // peer registration + peer->alloc_type = gds_peer::WQ; + peer->alloc_flags = GDS_ALLOC_WQ_DEFAULT | GDS_ALLOC_DBREC_DEFAULT; + if (flags & GDS_CREATE_QP_WQ_ON_GPU) { + gds_err("error, QP WQ on GPU is not supported yet\n"); + goto err; + } + if (flags & GDS_CREATE_QP_WQ_DBREC_ON_GPU) { + gds_warn("QP WQ DBREC on GPU\n"); + peer->alloc_flags |= GDS_ALLOC_DBREC_ON_GPU; + } + + exp_qp_attr = { + .send_cq = tx_gmexpcq->gcq.cq, + .recv_cq = rx_gmexpcq->gcq.cq, + .pd = pd, + .comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_PEER_DIRECT, + .peer_direct_attrs = peer_attr, + .qp_type = qp_attr->qp_type + }; + + assert(sizeof(exp_qp_attr.cap) == sizeof(qp_attr->cap)); + + memcpy(&exp_qp_attr.cap, &qp_attr->cap, sizeof(qp_attr->cap)); + + qp = ibv_exp_create_qp(context, qp_attr); + if (!qp) { + ret = EINVAL; + gds_err("error in ibv_exp_create_qp\n"); + goto err; + } + + gmexpqp->gqp.qp = qp; + gmexpqp->gqp.send_cq = tx_gmexpcq->gcq; + gmexpqp->gqp.recv_cq = rx_gmexpcq->gcq; + + gds_dbg("created gds_mlx5_exp_qp=%p\n", gmexpqp); + + return gmexpqp; + +err: + gds_dbg("destroying QP\n"); + gds_mlx5_exp_destroy_qp(gmexpqp); + + return NULL; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp) +{ + int retcode = 0; + int ret; + + if (!gmexpqp) + return retcode; + + assert(gmexpqp->gqp.dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + if (gmexpqp->gqp.qp) { + ret = ibv_destroy_qp(gmexpqp->gqp.qp); + if (ret) { + gds_err("error %d in destroy_qp\n", ret); + retcode = ret; + } + } + + if (gmexpqp->gqp.send_cq) { + ret = gds_destroy_cq(gmexpqp->gqp.send_cq); + if (ret) { + gds_err("error %d in destroy_cq send_cq\n", ret); + retcode = ret; + } + } + + if (gmexpqp->gqp.recv_cq) { + ret = gds_destroy_cq(gmexpqp->gqp.recv_cq); + if (ret) { + gds_err("error %d in destroy_cq recv_cq\n", ret); + retcode = ret; + } + } + + if (gmexpqp->res_domain) { + struct ibv_exp_destroy_res_domain_attr attr = {0,}; //IBV_EXP_DESTROY_RES_DOMAIN_RESERVED + ret = ibv_exp_destroy_res_domain(gmexpqp->gqp.dev_context, gmexpqp->res_domain, &attr); + if (ret) { + gds_err("ibv_exp_destroy_res_domain error %d: %s\n", ret, strerror(ret)); + retcode = ret; + } + } + + free(gmexpqp); + + return retcode; +} + diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp new file mode 100644 index 0000000..214342a --- /dev/null +++ b/src/mlx5-exp.hpp @@ -0,0 +1,20 @@ +typedef struct gds_mlx5_exp_cq { + gds_cq_t gcq; + ibv_exp_res_domain *res_domain +} gds_mlx5_exp_cq_t; + +typedef struct gds_mlx5_exp_qp { + gds_qp_t gqp; + ibv_exp_res_domain *res_domain +} gds_mlx5_exp_qp_t; + +static inline gds_mlx5_exp_cq_t *to_gds_mexp_cq(gds_cq_t *gcq) { + assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + return container_of(gcq, gds_mlx5_exp_cq_t, gcq); +} + +static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { + assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + return container_of(gqp, gds_mlx5_exp_qp_t, gqp); +} + From 5e789db4f759d8b0ce94bbf8f6834aa7ce1f7588 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 02:12:31 -0400 Subject: [PATCH 07/39] Implemented gds_mlx5_exp_destroy_cq --- src/mlx5-exp.cpp | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index 9e4efe4..47dc80a 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -28,8 +28,7 @@ static ibv_exp_res_domain *gds_mlx5_exp_create_res_domain(struct ibv_context *co //----------------------------------------------------------------------------- -static struct gds_mlx5_exp_cq_t * -gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, +gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, struct ibv_exp_res_domain *res_domain) @@ -89,7 +88,7 @@ gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, //----------------------------------------------------------------------------- -struct gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, +gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) { int ret = 0; @@ -243,3 +242,29 @@ int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp) return retcode; } +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq) +{ + int retcode = 0; + int ret; + + if (!gmexpcq) + return retcode; + + assert(gmexpcq->gcq.dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + if (gmexpcq->gcq.cq) { + ret = ibv_destroy_cq(gmexpcq->gcq.cq); + if (ret) { + gds_err("error %d in destroy_cq\n", ret); + retcode = ret; + } + } + + // res_domain will be destroyed in gds_mlx5_exp_destroy_qp. + + free(gmexpcq); + + return retcode; +} From 2a8ac7d93bc93e961626c5f4e240945d53a7042b Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 02:19:03 -0400 Subject: [PATCH 08/39] Implemented gds_destroy_qp and gds_destroy_cq by connecting to gds_mlx5_exp_* --- src/gdsync.cpp | 61 ++++++++++++++++++++---------------------------- src/mlx5-exp.hpp | 13 +++++++++++ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 5a14c51..862c0da 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1785,51 +1785,40 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, //----------------------------------------------------------------------------- -int gds_destroy_qp(struct gds_qp *gqp) +int gds_destroy_cq(struct gds_cq *gcq) { int retcode = 0; int ret; - if(!gqp) return retcode; + if (!gcq) + return retcode; - if(gqp->qp) - { - ret = ibv_destroy_qp(gqp->qp); - if (ret) { - gds_err("error %d in destroy_qp\n", ret); - retcode = ret; - } - } + // Currently, we support only exp-verbs. + assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); - if(gqp->send_cq.cq) - { - ret = ibv_destroy_cq(gqp->send_cq.cq); - if (ret) { - gds_err("error %d in destroy_cq send_cq\n", ret); - retcode = ret; - } - } + gds_mlx5_exp_cq_t *gmexpcq = to_gds_mexp_cq(gcq); - if(gqp->recv_cq.cq) - { - ret = ibv_destroy_cq(gqp->recv_cq.cq); - if (ret) { - gds_err("error %d in destroy_cq recv_cq\n", ret); - retcode = ret; - } - } + retcode = gds_mlx5_exp_destroy_cq(gmexpcq); - if(gqp->res_domain) { - struct ibv_exp_destroy_res_domain_attr attr; //IBV_EXP_DESTROY_RES_DOMAIN_RESERVED - attr.comp_mask=0; - ret = ibv_exp_destroy_res_domain(gqp->dev_context, gqp->res_domain, &attr); - if (ret) { - gds_err("ibv_exp_destroy_res_domain error %d: %s\n", ret, strerror(ret)); - retcode = ret; - } - } + return retcode; +} + +//----------------------------------------------------------------------------- + +int gds_destroy_qp(struct gds_qp *gqp) +{ + int retcode = 0; + int ret; + + if (!gqp) + return retcode; + + // Currently, we support only exp-verbs. + assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + gds_mlx5_exp_qp_t *gmexpqp = to_gds_mexp_qp(gqp); - free(gqp); + retcode = gds_mlx5_exp_destroy_qp(gmexpqp); return retcode; } diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index 214342a..0f525ee 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -1,3 +1,6 @@ +#include +#include + typedef struct gds_mlx5_exp_cq { gds_cq_t gcq; ibv_exp_res_domain *res_domain @@ -18,3 +21,13 @@ static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } +gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, + struct ibv_exp_res_domain *res_domain); + +gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, + gds_qp_init_attr_t *qp_attr, int gpu_id, int flags); + +int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq); +int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); From a9351f2bfabd7c1d44adf29c4cba10f73c615744 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 03:11:13 -0400 Subject: [PATCH 09/39] Reimplemented gds_create_qp by connecting to gds_mlx5_exp_create_qp --- src/gdsync.cpp | 80 ++++++++---------------------------------------- src/mlx5-exp.cpp | 60 +++++++++++++++--------------------- src/mlx5-exp.hpp | 14 +++++---- src/utils.hpp | 4 +++ 4 files changed, 48 insertions(+), 110 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 862c0da..56d2743 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1682,16 +1682,16 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) { int ret = 0; - struct gds_qp *gqp = NULL; - struct ibv_qp *qp = NULL; - struct gds_cq *rx_gcq = NULL, *tx_gcq = NULL; + gds_mlx5_exp_qp_t *gmexpqp = NULL; gds_peer *peer = NULL; gds_peer_attr *peer_attr = NULL; + gds_driver_type dtype; int old_errno = errno; gds_dbg("pd=%p context=%p gpu_id=%d flags=%08x current errno=%d\n", pd, context, gpu_id, flags, errno); assert(pd); assert(context); + assert(context->device); assert(qp_attr); if (flags & ~(GDS_CREATE_QP_WQ_ON_GPU|GDS_CREATE_QP_TX_CQ_ON_GPU|GDS_CREATE_QP_RX_CQ_ON_GPU|GDS_CREATE_QP_WQ_DBREC_ON_GPU)) { @@ -1699,87 +1699,31 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, return NULL; } - gqp = (struct gds_qp*)calloc(1, sizeof(struct gds_qp)); - if (!gqp) { - gds_err("cannot allocate memory\n"); - return NULL; - } - - gqp->dev_context=context; - // peer registration gds_dbg("before gds_register_peer_ex\n"); ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); if (ret) { - gds_err("error %d in gds_register_peer_ex\n", ret); - goto err; - } - - gqp->res_domain = gds_create_res_domain(context); - if (gqp->res_domain) - gds_dbg("using gqp->res_domain %p\n", gqp->res_domain); - else - gds_warn("NOT using gqp->res_domain\n"); - - tx_gcq = gds_create_cq_internal(context, qp_attr->cap.max_send_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gqp->res_domain); - if (!tx_gcq) { - ret = errno; - gds_err("error %d while creating TX CQ, old_errno=%d\n", ret, old_errno); + gds_err("error %d in gds_register_peer_ex\n", ret); goto err; } - rx_gcq = gds_create_cq_internal(context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gqp->res_domain); - if (!rx_gcq) { - ret = errno; - gds_err("error %d while creating RX CQ\n", ret); + dtype = gds_get_driver_type(context->device); + if (dtype != GDS_DRIVER_TYPE_MLX5_EXP) { + gds_err("Unsupported IB device\n"); goto err; } - // peer registration - qp_attr->send_cq = tx_gcq->cq; - qp_attr->recv_cq = rx_gcq->cq; - qp_attr->pd = pd; - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_PD; - - peer->alloc_type = gds_peer::WQ; - peer->alloc_flags = GDS_ALLOC_WQ_DEFAULT | GDS_ALLOC_DBREC_DEFAULT; - if (flags & GDS_CREATE_QP_WQ_ON_GPU) { - gds_err("error, QP WQ on GPU is not supported yet\n"); - goto err; - } - if (flags & GDS_CREATE_QP_WQ_DBREC_ON_GPU) { - gds_warn("QP WQ DBREC on GPU\n"); - peer->alloc_flags |= GDS_ALLOC_DBREC_ON_GPU; - } - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_PEER_DIRECT; - qp_attr->peer_direct_attrs = peer_attr; - - qp = ibv_exp_create_qp(context, qp_attr); - if (!qp) { - ret = EINVAL; - gds_err("error in ibv_exp_create_qp\n"); + gmexpqp = gds_mlx5_exp_create_qp(pd, context, qp_attr, peer, peer_attr, flags); + if (!gmexpqp) { + gds_err("Error in gds_mlx5_exp_create_qp.\n"); goto err; } - gqp->qp = qp; - gqp->send_cq.cq = qp->send_cq; - gqp->send_cq.curr_offset = 0; - gqp->recv_cq.cq = qp->recv_cq; - gqp->recv_cq.curr_offset = 0; - gqp->dtype = GDS_DRIVER_TYPE_MLX5_EXP; - - gds_dbg("created gds_qp=%p\n", gqp); + gds_dbg("created gds_qp=%p\n", gmexpqp->gqp); - return gqp; + return gmexpqp->gqp; err: - gds_dbg("destroying QP\n"); - gds_destroy_qp(gqp); - return NULL; } diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index 47dc80a..b39fa16 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -28,10 +28,11 @@ static ibv_exp_res_domain *gds_mlx5_exp_create_res_domain(struct ibv_context *co //----------------------------------------------------------------------------- -gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, - struct ibv_exp_res_domain *res_domain) +gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( + struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector, gds_peer *peer, gds_peer_attr *peer_attr, gds_alloc_cq_flags_t flags, + struct ibv_exp_res_domain *res_domain) { struct gds_mlx5_exp_cq_t *gmexpcq = NULL; ibv_exp_cq_init_attr attr; @@ -39,11 +40,9 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, gds_peer_attr *peer_attr = NULL; int ret = 0; - if (!context) - { - gds_dbg("Invalid input context\n"); - return NULL; - } + assert(context); + assert(peer); + assert(peer_attr); gmexpcq = (gds_mlx5_exp_cq_t *)calloc(1, sizeof(gds_mlx5_exp_cq_t)); if (!gmexpcq) { @@ -51,16 +50,6 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, return NULL; } - //Here we need to recover peer and peer_attr pointers to set alloc_type and alloc_flags - //before ibv_exp_create_cq - ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); - if (ret) { - gds_err("error %d while registering GPU peer\n", ret); - return NULL; - } - assert(peer); - assert(peer_attr); - peer->alloc_type = gds_peer::CQ; peer->alloc_flags = flags; @@ -88,8 +77,9 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, //----------------------------------------------------------------------------- -gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, - gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) +gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( + struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, + gds_peer *peer, gds_peer_attr *peer_attr, int flags) { int ret = 0; gds_mlx5_exp_qp_t *gmexpqp = NULL; @@ -103,8 +93,10 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context assert(pd); assert(context); assert(qp_attr); + assert(peer); + assert(peer_attr); - gmexpqp = (gds_mlx5_exp_qp_t *)calloc(1, sizeof(struct gds_qp)); + gmexpqp = (gds_mlx5_exp_qp_t *)calloc(1, sizeof(gds_mlx5_exp_qp_t)); if (!gqp) { gds_err("cannot allocate memory\n"); return NULL; @@ -113,32 +105,28 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context gmexpqp->gqp.dev_context = context; - // peer registration - gds_dbg("before gds_register_peer_ex\n"); - ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); - if (ret) { - gds_err("error %d in gds_register_peer_ex\n", ret); - goto err; - } - gmexpqp->res_domain = gds_create_res_domain(context); if (gmexpqp->res_domain) gds_dbg("using res_domain %p\n", gmexpqp->res_domain); else gds_warn("NOT using res_domain\n"); - tx_gmexpcq = gds_mlx5_exp_create_cq(context, qp_attr->cap.max_send_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gmexpqp->res_domain); + tx_gmexpcq = gds_mlx5_exp_create_cq( + context, qp_attr->cap.max_send_wr, NULL, NULL, 0, peer, peer_attr, + (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, + gmexpqp->res_domain + ); if (!tx_gmexpcq) { ret = errno; gds_err("error %d while creating TX CQ, old_errno=%d\n", ret, old_errno); goto err; } - rx_gmexpcq = gds_mlx5_exp_create_cq(context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gmexpqp->res_domain); + rx_gmexpcq = gds_mlx5_exp_create_cq( + context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, peer, peer_attr, + (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, + gmexpqp->res_domain + ); if (!rx_gmexpcq) { ret = errno; gds_err("error %d while creating RX CQ\n", ret); diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index 0f525ee..20e94f3 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -21,13 +21,15 @@ static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } -gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, - struct ibv_exp_res_domain *res_domain); +gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( + struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector, gds_peer *peer, gds_peer_attr *peer_attr, gds_alloc_cq_flags_t flags, + struct ibv_exp_res_domain *res_domain); -gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp(struct ibv_pd *pd, struct ibv_context *context, - gds_qp_init_attr_t *qp_attr, int gpu_id, int flags); +gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( + struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, + gds_peer *peer, gds_peer_attr *peer_attr, int flags); int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq); int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); diff --git a/src/utils.hpp b/src/utils.hpp index de2aaa8..1bacbb6 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -238,6 +238,10 @@ static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) //----------------------------------------------------------------------------- +int gds_destroy_cq(struct gds_cq *gcq); + +//----------------------------------------------------------------------------- + /* * Local variables: * c-indent-level: 8 From e18866864b7da32f97bd2a73900cab7b552c453c Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 03:16:09 -0400 Subject: [PATCH 10/39] Moved gds_send_wr from exp-verbs to ib-verbs in gds_kernel_latency.c --- tests/gds_kernel_latency.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/gds_kernel_latency.c b/tests/gds_kernel_latency.c index 63875bf..428dadb 100644 --- a/tests/gds_kernel_latency.c +++ b/tests/gds_kernel_latency.c @@ -542,23 +542,22 @@ static int pp_post_gpu_send(struct pingpong_context *ctx, uint32_t qpn, CUstream .wr_id = PINGPONG_SEND_WRID, .sg_list = &list, .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, .wr = { .ud = { .ah = ctx->ah, .remote_qpn = qpn, .remote_qkey = 0x11111111 } - }, - .comp_mask = 0 + } }; #if 0 if (IBV_QPT_UD != gds_qpt) { memset(&ewr, 0, sizeof(ewr)); ewr.num_sge = 1; - ewr.exp_send_flags = IBV_EXP_SEND_SIGNALED; - ewr.exp_opcode = IBV_EXP_WR_SEND; + ewr.send_flags = IBV_SEND_SIGNALED; + ewr.opcode = IBV_WR_SEND; ewr.wr_id = PINGPONG_SEND_WRID; ewr.sg_list = &list; ewr.next = NULL; @@ -580,23 +579,22 @@ static int pp_prepare_gpu_send(struct pingpong_context *ctx, uint32_t qpn, gds_s .wr_id = PINGPONG_SEND_WRID, .sg_list = &list, .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, .wr = { .ud = { .ah = ctx->ah, .remote_qpn = qpn, .remote_qkey = 0x11111111 } - }, - .comp_mask = 0 + } }; if (IBV_QPT_UD != gds_qpt) { memset(&ewr, 0, sizeof(ewr)); ewr.num_sge = 1; - ewr.exp_send_flags = IBV_EXP_SEND_SIGNALED; - ewr.exp_opcode = IBV_EXP_WR_SEND; + ewr.send_flags = IBV_SEND_SIGNALED; + ewr.opcode = IBV_WR_SEND; ewr.wr_id = PINGPONG_SEND_WRID; ewr.sg_list = &list; ewr.next = NULL; From 07d03a035c53876f29e10ca4e7608ba96399d270 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 5 Aug 2021 03:20:27 -0400 Subject: [PATCH 11/39] Fixed compile issues in gdsync.cpp and mlx5-exp.hpp --- include/gdsync/core.h | 8 ++++---- src/gdsync.cpp | 3 ++- src/mlx5-exp.hpp | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index c438594..a74299d 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -64,19 +64,19 @@ typedef enum gds_driver_type { GDS_DRIVER_TYPE_MLX5_DEVX } gds_driver_type_t; -struct gds_cq { +typedef struct gds_cq { struct ibv_cq *cq; uint32_t curr_offset; gds_driver_type_t dtype; -}; +} gds_cq_t; -struct gds_qp { +typedef struct gds_qp { struct ibv_qp *qp; struct gds_cq *send_cq; struct gds_cq *recv_cq; struct ibv_context *dev_context; gds_driver_type_t dtype; -}; +} gds_qp_t; /* \brief: Create a peer-enabled QP attached to the specified GPU id. * diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 56d2743..23907fe 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -43,6 +43,7 @@ #include "archutils.h" #include "mlnxutils.h" #include "task_queue.hpp" +#include "mlx5-exp.hpp" //----------------------------------------------------------------------------- @@ -1721,7 +1722,7 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_dbg("created gds_qp=%p\n", gmexpqp->gqp); - return gmexpqp->gqp; + return &gmexpqp->gqp; err: return NULL; diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index 20e94f3..60bb1fd 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -1,14 +1,16 @@ #include #include +#include + typedef struct gds_mlx5_exp_cq { gds_cq_t gcq; - ibv_exp_res_domain *res_domain + ibv_exp_res_domain *res_domain; } gds_mlx5_exp_cq_t; typedef struct gds_mlx5_exp_qp { gds_qp_t gqp; - ibv_exp_res_domain *res_domain + ibv_exp_res_domain *res_domain; } gds_mlx5_exp_qp_t; static inline gds_mlx5_exp_cq_t *to_gds_mexp_cq(gds_cq_t *gcq) { @@ -17,7 +19,7 @@ static inline gds_mlx5_exp_cq_t *to_gds_mexp_cq(gds_cq_t *gcq) { } static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { - assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } From 173841f3174272eca05d55f7a8c838e7ecbaa01b Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 6 Aug 2021 00:54:30 -0400 Subject: [PATCH 12/39] Added mlx5-exp.cpp to the compile list --- Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index ac9a0cc..27fce57 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,13 +20,13 @@ EXTRA_DIST = autogen.sh include_HEADERS = include/gdsync.h libgdsyncincludedir = $(includedir)/gdsync -libgdsyncinclude_HEADERS = include/gdsync/core.h include/gdsync/device.cuh include/gdsync/mlx5.h include/gdsync/tools.h +libgdsyncinclude_HEADERS = include/gdsync/core.h include/gdsync/device.cuh include/gdsync/mlx5.h include/gdsync/tools.h src_libgdsync_la_CFLAGS = $(AM_CFLAGS) -src_libgdsync_la_SOURCES = src/gdsync.cpp src/memmgr.cpp src/mem.cpp src/objs.cpp src/apis.cpp src/mlx5.cpp include/gdsync.h +src_libgdsync_la_SOURCES = src/gdsync.cpp src/memmgr.cpp src/mem.cpp src/objs.cpp src/apis.cpp src/mlx5.cpp src/mlx5-exp.cpp include/gdsync.h src_libgdsync_la_LDFLAGS = -version-info @VERSION_INFO@ -noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h +noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h src/mlx5-exp.hpp # if enabled at configure time From 8d1ed503001c40dd6120b4d61c83f07b4395f631 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 6 Aug 2021 00:55:35 -0400 Subject: [PATCH 13/39] Moved gds_prepare_send to gds_mlx5_exp_* and fixed compile errors in mlx5-exp.* --- src/apis.cpp | 34 ++++++++++--------------- src/mlx5-exp.cpp | 64 ++++++++++++++++++++++++++++++++++-------------- src/mlx5-exp.hpp | 11 +++++++++ 3 files changed, 69 insertions(+), 40 deletions(-) diff --git a/src/apis.cpp b/src/apis.cpp index cd532d7..0801771 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -51,6 +51,7 @@ #include "utils.hpp" #include "archutils.h" #include "mlnxutils.h" +#include "mlx5-exp.hpp" //----------------------------------------------------------------------------- @@ -171,33 +172,24 @@ int gds_post_recv(struct gds_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr //----------------------------------------------------------------------------- -int gds_prepare_send(struct gds_qp *qp, gds_send_wr *p_ewr, +int gds_prepare_send(struct gds_qp *gqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request) { int ret = 0; + gds_mlx5_exp_qp_t *gmexpqp; + gds_init_send_info(request); - assert(qp); - assert(qp->qp); - ret = ibv_exp_post_send(qp->qp, p_ewr, bad_ewr); - if (ret) { + assert(gqp); + assert(gqp->qp); + assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + gmexpqp = to_gds_mexp_qp(gqp); + + ret = gds_mlx5_exp_prepare_send(gmexpqp, p_ewr, bad_ewr, request); + if (ret) + gds_err("Error %d in gds_mlx5_exp_prepare_send.\n", ret); - if (ret == ENOMEM) { - // out of space error can happen too often to report - gds_dbg("ENOMEM error %d in ibv_exp_post_send\n", ret); - } else { - gds_err("error %d in ibv_exp_post_send\n", ret); - } - goto out; - } - - ret = ibv_exp_peer_commit_qp(qp->qp, &request->commit); - if (ret) { - gds_err("error %d in ibv_exp_peer_commit_qp\n", ret); - //gds_wait_kernel(); - goto out; - } -out: return ret; } diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index b39fa16..f76cf12 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -34,10 +34,8 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( int comp_vector, gds_peer *peer, gds_peer_attr *peer_attr, gds_alloc_cq_flags_t flags, struct ibv_exp_res_domain *res_domain) { - struct gds_mlx5_exp_cq_t *gmexpcq = NULL; + gds_mlx5_exp_cq_t *gmexpcq = NULL; ibv_exp_cq_init_attr attr; - gds_peer *peer = NULL; - gds_peer_attr *peer_attr = NULL; int ret = 0; assert(context); @@ -85,9 +83,7 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( gds_mlx5_exp_qp_t *gmexpqp = NULL; struct ibv_qp *qp = NULL; gds_mlx5_exp_cq_t *rx_gmexpcq = NULL, *tx_gmexpcq = NULL; - gds_peer *peer = NULL; - gds_peer_attr *peer_attr = NULL; - struct ibv_qp_init_attr exp_qp_attr = {0,}; + struct ibv_exp_qp_init_attr exp_qp_attr = {0,}; int old_errno = errno; assert(pd); @@ -97,7 +93,7 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( assert(peer_attr); gmexpqp = (gds_mlx5_exp_qp_t *)calloc(1, sizeof(gds_mlx5_exp_qp_t)); - if (!gqp) { + if (!gmexpqp) { gds_err("cannot allocate memory\n"); return NULL; } @@ -105,7 +101,7 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( gmexpqp->gqp.dev_context = context; - gmexpqp->res_domain = gds_create_res_domain(context); + gmexpqp->res_domain = gds_mlx5_exp_create_res_domain(context); if (gmexpqp->res_domain) gds_dbg("using res_domain %p\n", gmexpqp->res_domain); else @@ -145,29 +141,30 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( peer->alloc_flags |= GDS_ALLOC_DBREC_ON_GPU; } - exp_qp_attr = { - .send_cq = tx_gmexpcq->gcq.cq, - .recv_cq = rx_gmexpcq->gcq.cq, - .pd = pd, - .comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_PEER_DIRECT, - .peer_direct_attrs = peer_attr, - .qp_type = qp_attr->qp_type - }; + exp_qp_attr.send_cq = tx_gmexpcq->gcq.cq; + exp_qp_attr.recv_cq = rx_gmexpcq->gcq.cq; + exp_qp_attr.pd = pd; + exp_qp_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_PEER_DIRECT; + exp_qp_attr.peer_direct_attrs = peer_attr; + exp_qp_attr.qp_type = qp_attr->qp_type; assert(sizeof(exp_qp_attr.cap) == sizeof(qp_attr->cap)); memcpy(&exp_qp_attr.cap, &qp_attr->cap, sizeof(qp_attr->cap)); - qp = ibv_exp_create_qp(context, qp_attr); + qp = ibv_exp_create_qp(context, &exp_qp_attr); if (!qp) { ret = EINVAL; gds_err("error in ibv_exp_create_qp\n"); goto err; } + tx_gmexpcq->gcq.cq = qp->send_cq; + rx_gmexpcq->gcq.cq = qp->recv_cq; + gmexpqp->gqp.qp = qp; - gmexpqp->gqp.send_cq = tx_gmexpcq->gcq; - gmexpqp->gqp.recv_cq = rx_gmexpcq->gcq; + gmexpqp->gqp.send_cq = &tx_gmexpcq->gcq; + gmexpqp->gqp.recv_cq = &rx_gmexpcq->gcq; gds_dbg("created gds_mlx5_exp_qp=%p\n", gmexpqp); @@ -256,3 +253,32 @@ int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq) return retcode; } + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, + gds_send_wr **bad_ewr, + gds_send_request_t *request) +{ + int ret = 0; + ret = ibv_post_send(gmexpqp->gqp.qp, p_ewr, bad_ewr); + if (ret) { + + if (ret == ENOMEM) { + // out of space error can happen too often to report + gds_dbg("ENOMEM error %d in ibv_post_send\n", ret); + } else { + gds_err("error %d in ibv_post_send\n", ret); + } + goto out; + } + + ret = ibv_exp_peer_commit_qp(gmexpqp->gqp.qp, &request->commit); + if (ret) { + gds_err("error %d in ibv_exp_peer_commit_qp\n", ret); + goto out; + } +out: + return ret; +} + diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index 60bb1fd..d289c83 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -1,8 +1,15 @@ +#include +#include +#include + #include #include #include +#include "objs.hpp" +#include "utils.hpp" + typedef struct gds_mlx5_exp_cq { gds_cq_t gcq; ibv_exp_res_domain *res_domain; @@ -35,3 +42,7 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq); int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); + +int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, + gds_send_wr **bad_ewr, + gds_send_request_t *request); From 10e164ec794c24df2a997a4f8db69385f7256ca9 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 6 Aug 2021 00:56:18 -0400 Subject: [PATCH 14/39] Modified gds_kernel_* applications to fit the new API/structs --- tests/gds_kernel_latency.c | 18 +++++++++--------- tests/gds_kernel_loopback_latency.c | 29 +++++++++++++---------------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/tests/gds_kernel_latency.c b/tests/gds_kernel_latency.c index 428dadb..04370f5 100644 --- a/tests/gds_kernel_latency.c +++ b/tests/gds_kernel_latency.c @@ -495,7 +495,7 @@ static int pp_wait_cq(struct pingpong_context *ctx, int is_client) { int ret; if (ctx->peersync) { - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); } else { if (is_client) { do { @@ -674,7 +674,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].tag = GDS_TAG_SEND; wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; @@ -683,7 +683,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].tag = GDS_TAG_WAIT; wdesc->descs[k].wait = &wdesc->wait_tx_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -713,14 +713,14 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("[%d] error %d in gds_stream_wait_cq\n", my_rank, ret); @@ -749,7 +749,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin if (ctx->use_desc_apis) { work_desc_t *wdesc = calloc(1, sizeof(*wdesc)); int k = 0; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -771,7 +771,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin CUCHECK(cuStreamAddCallback(gpu_stream, post_work_cb, wdesc, 0)); } } else if (ctx->peersync) { - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("error %d in gds_stream_wait_cq\n", ret); @@ -804,7 +804,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].tag = GDS_TAG_SEND; wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; @@ -833,7 +833,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); diff --git a/tests/gds_kernel_loopback_latency.c b/tests/gds_kernel_loopback_latency.c index b2d209c..f6ccc32 100644 --- a/tests/gds_kernel_loopback_latency.c +++ b/tests/gds_kernel_loopback_latency.c @@ -511,16 +511,15 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) .wr_id = PINGPONG_SEND_WRID, .sg_list = &list, .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, .wr = { .ud = { .ah = ctx->ah, .remote_qpn = qpn, .remote_qkey = 0x11111111 } - }, - .comp_mask = 0 + } }; gds_send_wr *bad_ewr; return gds_post_send(ctx->gds_qp, &ewr, &bad_ewr); @@ -538,16 +537,15 @@ static int pp_post_gpu_send(struct pingpong_context *ctx, uint32_t qpn, CUstream .wr_id = PINGPONG_SEND_WRID, .sg_list = &list, .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, .wr = { .ud = { .ah = ctx->ah, .remote_qpn = qpn, .remote_qkey = 0x11111111 } - }, - .comp_mask = 0 + } }; gds_send_wr *bad_ewr; return gds_stream_queue_send(*p_gpu_stream, ctx->gds_qp, &ewr, &bad_ewr); @@ -565,16 +563,15 @@ static int pp_prepare_gpu_send(struct pingpong_context *ctx, uint32_t qpn, gds_s .wr_id = PINGPONG_SEND_WRID, .sg_list = &list, .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, .wr = { .ud = { .ah = ctx->ah, .remote_qpn = qpn, .remote_qkey = 0x11111111 } - }, - .comp_mask = 0 + } }; gds_send_wr *bad_ewr; //printf("gpu_post_send_on_stream\n"); @@ -655,7 +652,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; @@ -665,7 +662,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].wait = &wdesc->wait_tx_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -697,7 +694,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin break; } - ret = gds_stream_wait_cq(gpu_stream_server, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream_server, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); @@ -705,7 +702,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin break; } - ret = gds_stream_wait_cq(gpu_stream_server, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream_server, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("error %d in gds_stream_wait_cq\n", ret); From b1cab0511afe6efe5227e00ed7e790e064884394 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 11 Aug 2021 04:06:38 -0400 Subject: [PATCH 15/39] Moved gds_wait_request to gds_mlx5_exp_wait_request and made the former an opaque struct --- include/gdsync/core.h | 7 +++++-- src/mlx5-exp.hpp | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index a74299d..30fc977 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -175,8 +175,11 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req */ typedef struct gds_wait_request { - struct ibv_exp_peer_peek peek; - struct peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; + gds_driver_type_t dtype; + uint8_t pad0[4]; + uint8_t reserved0[40]; + uint8_t reserved1[56 * GDS_WAIT_INFO_MAX_OPS]; + uint8_t pad1[16]; } gds_wait_request_t; /** diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index d289c83..a92e0e6 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -6,6 +6,7 @@ #include #include +#include #include "objs.hpp" #include "utils.hpp" @@ -20,6 +21,18 @@ typedef struct gds_mlx5_exp_qp { ibv_exp_res_domain *res_domain; } gds_mlx5_exp_qp_t; +typedef struct gds_mlx5_exp_wait_request { + gds_driver_type_t dtype; + uint8_t pad0[4]; + struct ibv_exp_peer_peek peek; + struct peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; + uint8_t pad1[16]; +} gds_mlx5_exp_wait_request_t; + +static_assert(sizeof(gds_mlx5_exp_wait_request_t) % 64 == 0, "gds_mlx5_exp_wait_request_t must be 64-byte aligned."); +static_assert(sizeof(gds_mlx5_exp_wait_request_t) <= sizeof(gds_wait_request_t), "The size of gds_mlx5_exp_wait_request_t must be less than or equal to that of gds_wait_request_t."); +static_assert(offsetof(gds_mlx5_exp_wait_request_t, dtype) == offsetof(gds_wait_request_t, dtype), "dtype of gds_mlx5_exp_wait_request_t and gds_wait_request_t must be at the same offset."); + static inline gds_mlx5_exp_cq_t *to_gds_mexp_cq(gds_cq_t *gcq) { assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return container_of(gcq, gds_mlx5_exp_cq_t, gcq); @@ -30,6 +43,19 @@ static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } +static inline gds_mlx5_exp_wait_request_t *to_gds_mexp_wait_request(gds_wait_request_t *gwreq) { + assert(gwreq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + return (gds_mlx5_exp_wait_request_t *)(gwreq); +} + +static inline const gds_mlx5_exp_wait_request_t *to_gds_mexp_wait_request(const gds_wait_request_t *gwreq) { + return (const gds_mlx5_exp_wait_request_t *)to_gds_mexp_wait_request((const gds_wait_request_t *)gwreq); +} + +static inline uint32_t gds_mlx5_exp_get_num_wait_request_entries(gds_mlx5_exp_wait_request_t *gmexp_request) { + return gmexp_request->peek.entries; +} + gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, @@ -46,3 +72,12 @@ int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request); + +void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset); +void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t count); +int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_request_t *request, int flags); +int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t *dw, uint32_t val); +int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_request_t *request); +int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_mlx5_exp_wait_request_t *request, gds_op_list_t ¶ms, int flags); +int gds_mlx5_exp_post_wait_descriptor(gds_mlx5_exp_wait_request_t *request, int flags); +int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp_wait_request_t *request); From 17b246f2ec485f97d324a36f6937b8c02dd646a4 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Wed, 11 Aug 2021 04:07:49 -0400 Subject: [PATCH 16/39] Moved gds_wait_request related functions to mlx5-exp --- src/apis.cpp | 88 ++++++--------- src/gdsync.cpp | 71 +----------- src/mlx5-exp.cpp | 280 +++++++++++++++++++++++++++++++++++++++++++++++ src/mlx5.cpp | 104 +----------------- src/utils.hpp | 2 + 5 files changed, 323 insertions(+), 222 deletions(-) diff --git a/src/apis.cpp b/src/apis.cpp index 0801771..e9ad2ed 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -56,7 +56,7 @@ //----------------------------------------------------------------------------- -static void gds_init_ops(struct peer_op_wr *op, int count) +void gds_init_ops(struct peer_op_wr *op, int count) { int i = count; while (--i) @@ -80,13 +80,15 @@ static void gds_init_send_info(gds_send_request_t *info) static void gds_init_wait_request(gds_wait_request_t *request, uint32_t offset) { + gds_mlx5_exp_wait_request_t *gmexp_request; gds_dbg("wait_request=%p offset=%08x\n", request, offset); memset(request, 0, sizeof(*request)); - request->peek.storage = request->wr; - request->peek.entries = sizeof(request->wr)/sizeof(request->wr[0]); - request->peek.whence = IBV_EXP_PEER_PEEK_ABSOLUTE; - request->peek.offset = offset; - gds_init_ops(request->peek.storage, request->peek.entries); + + request->dtype = GDS_DRIVER_TYPE_MLX5_EXP; + + gmexp_request = to_gds_mexp_wait_request(request); + + gds_mlx5_exp_init_wait_request(gmexp_request, offset); } //----------------------------------------------------------------------------- @@ -273,7 +275,9 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flags) { - int retcode = 0; + gds_mlx5_exp_cq_t *gmexpcq; + gds_mlx5_exp_wait_request_t *gmexp_request; + if (flags != 0) { gds_err("invalid flags != 0\n"); return EINVAL; @@ -281,51 +285,19 @@ int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flag gds_init_wait_request(request, cq->curr_offset++); - retcode = ibv_exp_peer_peek_cq(cq->cq, &request->peek); - if (retcode == -ENOSPC) { - // TODO: handle too few entries - gds_err("not enough ops in peer_peek_cq\n"); - goto out; - } else if (retcode) { - gds_err("error %d in peer_peek_cq\n", retcode); - goto out; - } - //gds_dump_wait_request(request, 1); - out: - return retcode; + gmexpcq = to_gds_mexp_cq(cq); + gmexp_request = to_gds_mexp_wait_request(request); + + return gds_mlx5_exp_prepare_wait_cq(gmexpcq, gmexp_request, flags); } //----------------------------------------------------------------------------- int gds_append_wait_cq(gds_wait_request_t *request, uint32_t *dw, uint32_t val) { - int ret = 0; - unsigned MAX_NUM_ENTRIES = sizeof(request->wr)/sizeof(request->wr[0]); - unsigned n = request->peek.entries; - struct peer_op_wr *wr = request->peek.storage; - - if (n + 1 > MAX_NUM_ENTRIES) { - gds_err("no space left to stuff a poke\n"); - ret = ENOMEM; - goto out; - } + gds_mlx5_exp_wait_request_t *gmexp_request = to_gds_mexp_wait_request(request); - // at least 1 op - assert(n); - assert(wr); - - for (; n; --n) wr = wr->next; - assert(wr); - - wr->type = IBV_EXP_PEER_OP_STORE_DWORD; - wr->wr.dword_va.data = val; - wr->wr.dword_va.target_id = 0; // direct mapping, offset IS the address - wr->wr.dword_va.offset = (ptrdiff_t)(dw-(uint32_t*)0); - - ++request->peek.entries; - - out: - return ret; + return gds_mlx5_exp_append_wait_cq(gmexp_request, dw, val); } //----------------------------------------------------------------------------- @@ -346,12 +318,16 @@ int gds_stream_post_wait_cq_all(CUstream stream, int count, gds_wait_request_t * static int gds_abort_wait_cq(struct gds_cq *cq, gds_wait_request_t *request) { + gds_mlx5_exp_cq_t *gmexpcq; + gds_mlx5_exp_wait_request_t *gmexp_request; + assert(cq); assert(request); - struct ibv_exp_peer_abort_peek abort_ctx; - abort_ctx.peek_id = request->peek.peek_id; - abort_ctx.comp_mask = 0; - return ibv_exp_peer_abort_peek_cq(cq->cq, &abort_ctx); + + gmexpcq = to_gds_mexp_cq(cq); + gmexp_request = to_gds_mexp_wait_request(request); + + return gds_mlx5_exp_abort_wait_cq(gmexpcq, gmexp_request); } //----------------------------------------------------------------------------- @@ -549,7 +525,7 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem n_mem_ops += desc->send->commit.entries + 2; // extra space, ugly break; case GDS_TAG_WAIT: - n_mem_ops += desc->wait->peek.entries + 2; // ditto + n_mem_ops += gds_mlx5_exp_get_num_wait_request_entries(to_gds_mexp_wait_request(desc->wait)) + 2; // ditto break; case GDS_TAG_WAIT_VALUE32: case GDS_TAG_WRITE_VALUE32: @@ -618,15 +594,15 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ break; } case GDS_TAG_WAIT: { - gds_wait_request_t *wreq = desc->wait; + gds_mlx5_exp_wait_request_t *wreq = to_gds_mexp_wait_request(desc->wait); int flags = 0; if (move_flush && i != last_wait) { gds_dbg("discarding FLUSH!\n"); flags = GDS_POST_OPS_DISCARD_WAIT_FLUSH; } - retcode = gds_post_ops(peer, wreq->peek.entries, wreq->peek.storage, params, flags); + retcode = gds_mlx5_exp_stream_post_wait_descriptor(peer, wreq, params, flags); if (retcode) { - gds_err("error %d in gds_post_ops\n", retcode); + gds_err("error %d in gds_mlx5_exp_stream_post_wait_descriptor\n", retcode); ret = retcode; goto out; } @@ -697,10 +673,10 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) } case GDS_TAG_WAIT: { gds_dbg("desc[%zu] WAIT\n", i); - gds_wait_request_t *wreq = desc->wait; - retcode = gds_post_ops_on_cpu(wreq->peek.entries, wreq->peek.storage, flags); + gds_mlx5_exp_wait_request_t *wreq = to_gds_mexp_wait_request(desc->wait); + retcode = gds_mlx5_exp_post_wait_descriptor(wreq, flags); if (retcode) { - gds_err("error %d in gds_post_ops_on_cpu\n", retcode); + gds_err("error %d in gds_mlx5_exp_post_wait_descriptor\n", retcode); ret = retcode; goto out; } diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 23907fe..670bc45 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1169,73 +1169,14 @@ int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uin //----------------------------------------------------------------------------- -static void gds_dump_ops(struct peer_op_wr *op, size_t count) -{ - size_t n = 0; - for (; op; op = op->next, ++n) { - gds_dbg("op[%zu] type:%d\n", n, op->type); - switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - gds_dbg("STORE_QWORD data:%x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.dword_va.data, op->wr.dword_va.target_id, - op->wr.dword_va.offset, dev_ptr); - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - gds_dbg("STORE_QWORD data:%" PRIx64 " target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.qword_va.data, op->wr.qword_va.target_id, - op->wr.qword_va.offset, dev_ptr); - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - gds_dbg("COPY_BLOCK src:%p len:%zu target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.copy_op.src, op->wr.copy_op.len, - op->wr.copy_op.target_id, op->wr.copy_op.offset, - dev_ptr); - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - gds_dbg("%s data:%08x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - (op->type==IBV_EXP_PEER_OP_POLL_AND_DWORD) ? "POLL_AND_DW" : "POLL_NOR_SDW", - op->wr.dword_va.data, - op->wr.dword_va.target_id, - op->wr.dword_va.offset, - dev_ptr); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - break; - } - } - - assert(count == n); -} - -//----------------------------------------------------------------------------- - void gds_dump_wait_request(gds_wait_request_t *request, size_t count) { - for (size_t j=0; jentries, peek->whence, peek->offset, - peek->peek_id, peek->comp_mask); - gds_dump_ops(peek->storage, peek->entries); - } + gds_mlx5_exp_wait_request_t *gmexp_request; + if (count == 0) + return; + + gmexp_request = to_gds_mexp_wait_request(request); + gds_mlx5_exp_dump_wait_request(gmexp_request, count); } //----------------------------------------------------------------------------- diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index f76cf12..8d7e939 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -282,3 +282,283 @@ int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, return ret; } +//----------------------------------------------------------------------------- + +void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset) +{ + gds_dbg("wait_request=%p offset=%08x\n", request, offset); + request->peek.storage = request->wr; + request->peek.entries = sizeof(request->wr)/sizeof(request->wr[0]); + request->peek.whence = IBV_EXP_PEER_PEEK_ABSOLUTE; + request->peek.offset = offset; + gds_init_ops(request->peek.storage, request->peek.entries); +} + +//----------------------------------------------------------------------------- + +static void gds_mlx5_exp_dump_ops(struct peer_op_wr *op, size_t count) +{ + size_t n = 0; + for (; op; op = op->next, ++n) { + gds_dbg("op[%zu] type:%d\n", n, op->type); + switch(op->type) { + case IBV_EXP_PEER_OP_FENCE: { + gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + break; + } + case IBV_EXP_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + gds_dbg("STORE_QWORD data:%x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.dword_va.data, op->wr.dword_va.target_id, + op->wr.dword_va.offset, dev_ptr); + break; + } + case IBV_EXP_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + gds_dbg("STORE_QWORD data:%" PRIx64 " target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.qword_va.data, op->wr.qword_va.target_id, + op->wr.qword_va.offset, dev_ptr); + break; + } + case IBV_EXP_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + gds_dbg("COPY_BLOCK src:%p len:%zu target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.copy_op.src, op->wr.copy_op.len, + op->wr.copy_op.target_id, op->wr.copy_op.offset, + dev_ptr); + break; + } + case IBV_EXP_PEER_OP_POLL_AND_DWORD: + case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + gds_dbg("%s data:%08x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + (op->type==IBV_EXP_PEER_OP_POLL_AND_DWORD) ? "POLL_AND_DW" : "POLL_NOR_SDW", + op->wr.dword_va.data, + op->wr.dword_va.target_id, + op->wr.dword_va.offset, + dev_ptr); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + break; + } + } + + assert(count == n); +} + +//----------------------------------------------------------------------------- + +void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t count) +{ + for (size_t j = 0; j < count; ++j) { + struct ibv_exp_peer_peek *peek = &request[j].peek; + gds_dbg("req[%zu] entries:%u whence:%u offset:%u peek_id:%" PRIx64 " comp_mask:%08x\n", + j, peek->entries, peek->whence, peek->offset, + peek->peek_id, peek->comp_mask); + gds_mlx5_exp_dump_ops(peek->storage, peek->entries); + } +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_request_t *request, int flags) +{ + int retcode = 0; + + retcode = ibv_exp_peer_peek_cq(mexpcq->gcq.cq, &request->peek); + if (retcode == ENOSPC) { + // TODO: handle too few entries + gds_err("not enough ops in peer_peek_cq\n"); + goto out; + } else if (retcode) { + gds_err("error %d in peer_peek_cq\n", retcode); + goto out; + } + //gds_dump_wait_request(request, 1); + out: + return retcode; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t *dw, uint32_t val) +{ + int ret = 0; + unsigned MAX_NUM_ENTRIES = sizeof(request->wr) / sizeof(request->wr[0]); + unsigned n = request->peek.entries; + struct peer_op_wr *wr = request->peek.storage; + + if (n + 1 > MAX_NUM_ENTRIES) { + gds_err("no space left to stuff a poke\n"); + ret = ENOMEM; + goto out; + } + + // at least 1 op + assert(n); + assert(wr); + + for (; n; --n) + wr = wr->next; + + assert(wr); + + wr->type = IBV_EXP_PEER_OP_STORE_DWORD; + wr->wr.dword_va.data = val; + wr->wr.dword_va.target_id = 0; // direct mapping, offset IS the address + wr->wr.dword_va.offset = (ptrdiff_t)(dw-(uint32_t*)0); + + ++request->peek.entries; + +out: + return ret; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_request_t *request) +{ + struct ibv_exp_peer_abort_peek abort_ctx; + abort_ctx.peek_id = request->peek.peek_id; + abort_ctx.comp_mask = 0; + return ibv_exp_peer_abort_peek_cq(gmexpcq->gcq.cq, &abort_ctx); +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_mlx5_exp_wait_request_t *request, gds_op_list_t ¶ms, int flags) +{ + int ret = 0; + + ret = gds_post_ops(peer, request->peek.entries, request->peek.storage, params, flags); + if (ret) + gds_err("error %d in gds_post_ops\n", ret); + + return ret; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_post_wait_descriptor(gds_mlx5_exp_wait_request_t *request, int flags) +{ + int ret = 0; + + ret = gds_post_ops_on_cpu(request->peek.entries, request->peek.storage, flags); + if (ret) + gds_err("error %d in gds_post_ops_on_cpu\n", ret); + + return ret; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp_wait_request_t *request) +{ + int retcode = 0; + size_t n_ops = request->peek.entries; + peer_op_wr *op = request->peek.storage; + size_t n = 0; + + memset(mlx5_i, 0, sizeof(*mlx5_i)); + + for (; op && n < n_ops; op = op->next, ++n) { + switch(op->type) { + case IBV_EXP_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + break; + } + if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + case IBV_EXP_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + if (n != 1) { + gds_err("store DWORD is not 2nd op\n"); + retcode = EINVAL; + break; + } + mlx5_i->flag_ptr = (uint32_t*)dev_ptr; + mlx5_i->flag_value = data; + break; + } + case IBV_EXP_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); + gds_err("unsupported QWORD op\n"); + retcode = EINVAL; + break; + } + case IBV_EXP_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + gds_err("unsupported COPY_BLOCK\n"); + retcode = EINVAL; + break; + } + case IBV_EXP_PEER_OP_POLL_AND_DWORD: + case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: + case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + + gds_dbg("OP_POLL_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + + mlx5_i->cqe_ptr = (uint32_t *)dev_ptr; + mlx5_i->cqe_value = data; + + switch(op->type) { + case IBV_EXP_PEER_OP_POLL_NOR_DWORD: + // GPU SMs can always do NOR + mlx5_i->cond = GDS_WAIT_COND_NOR; + break; + case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: + mlx5_i->cond = GDS_WAIT_COND_GEQ; + break; + case IBV_EXP_PEER_OP_POLL_AND_DWORD: + mlx5_i->cond = GDS_WAIT_COND_AND; + break; + default: + gds_err("unexpected op type\n"); + retcode = EINVAL; + goto err; + } + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + err: + if (retcode) { + gds_err("error in fill func at entry n=%zu\n", n); + break; + } + } + return retcode; +} diff --git a/src/mlx5.cpp b/src/mlx5.cpp index a2c7b39..b026374 100644 --- a/src/mlx5.cpp +++ b/src/mlx5.cpp @@ -40,6 +40,7 @@ //#include "mem.hpp" #include "objs.hpp" #include "utils.hpp" +#include "mlx5-exp.hpp" #if 0 union { uint64_t qw; uint32_t dw[2]; } db_val; @@ -180,107 +181,8 @@ int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_ml int gds_mlx5_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *request) { - int retcode = 0; - size_t n_ops = request->peek.entries; - peer_op_wr *op = request->peek.storage; - size_t n = 0; - - memset(mlx5_i, 0, sizeof(*mlx5_i)); - - for (; op && n < n_ops; op = op->next, ++n) { - switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); - break; - } - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; - break; - } - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - if (n != 1) { - gds_err("store DWORD is not 2nd op\n"); - retcode = EINVAL; - break; - } - mlx5_i->flag_ptr = (uint32_t*)dev_ptr; - mlx5_i->flag_value = data; - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); - gds_err("unsupported QWORD op\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - gds_err("unsupported COPY_BLOCK\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - - gds_dbg("OP_POLL_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - - mlx5_i->cqe_ptr = (uint32_t *)dev_ptr; - mlx5_i->cqe_value = data; - - switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: - // GPU SMs can always do NOR - mlx5_i->cond = GDS_WAIT_COND_NOR; - break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - mlx5_i->cond = GDS_WAIT_COND_GEQ; - break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - mlx5_i->cond = GDS_WAIT_COND_AND; - break; - default: - gds_err("unexpected op type\n"); - retcode = EINVAL; - goto err; - } - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - err: - if (retcode) { - gds_err("error in fill func at entry n=%zu\n", n); - break; - } - } - return retcode; + const gds_mlx5_exp_wait_request_t *gmexp_request = to_gds_mexp_wait_request(request); + return gds_mlx5_exp_get_wait_descs(mlx5_i, gmexp_request); } //----------------------------------------------------------------------------- diff --git a/src/utils.hpp b/src/utils.hpp index 1bacbb6..c5d0774 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -240,6 +240,8 @@ static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) int gds_destroy_cq(struct gds_cq *gcq); +void gds_init_ops(struct peer_op_wr *op, int count); + //----------------------------------------------------------------------------- /* From 6be27ac863da603a2d7928b07e59750b6e02f5d0 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 04:41:13 -0400 Subject: [PATCH 17/39] Changed the definition of gds_send_request_t --- include/gdsync/core.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index 30fc977..fa213d3 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -161,9 +161,13 @@ enum { */ typedef struct gds_send_request { - struct ibv_exp_peer_commit commit; - struct peer_op_wr wr[GDS_SEND_INFO_MAX_OPS]; + gds_driver_type_t dtype; + uint8_t pad0[4]; + uint8_t reserved0[32]; + uint8_t reserved1[56 * GDS_SEND_INFO_MAX_OPS]; + uint8_t pad1[24]; } gds_send_request_t; +static_assert(sizeof(gds_send_request_t) % 64 == 0, "gds_send_request_t must be 64-byte aligned."); int gds_prepare_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request); int gds_stream_post_send(CUstream stream, gds_send_request_t *request); @@ -181,6 +185,7 @@ typedef struct gds_wait_request { uint8_t reserved1[56 * GDS_WAIT_INFO_MAX_OPS]; uint8_t pad1[16]; } gds_wait_request_t; +static_assert(sizeof(gds_wait_request_t) % 64 == 0, "gds_wait_request_t must be 64-byte aligned."); /** * Initializes a wait request out of the next heading CQE, which is kept in From f02657839f251cd89d2ba4bc37e511cc464bfac1 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 04:41:56 -0400 Subject: [PATCH 18/39] Added gds_mlx5_exp_send_request_t definition and supported functions --- src/mlx5-exp.hpp | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index a92e0e6..7e4ba14 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -21,6 +21,17 @@ typedef struct gds_mlx5_exp_qp { ibv_exp_res_domain *res_domain; } gds_mlx5_exp_qp_t; +typedef struct gds_mlx5_exp_send_request { + gds_driver_type_t dtype; + uint8_t pad0[4]; + struct ibv_exp_peer_commit commit; + struct peer_op_wr wr[GDS_SEND_INFO_MAX_OPS]; + uint8_t pad1[24]; +} gds_mlx5_exp_send_request_t; +static_assert(sizeof(gds_mlx5_exp_send_request_t) % 64 == 0, "gds_mlx5_exp_send_request_t must be 64-byte aligned."); +static_assert(sizeof(gds_mlx5_exp_send_request_t) <= sizeof(gds_send_request_t), "The size of gds_mlx5_exp_send_request_t must be less than or equal to that of gds_send_request_t."); +static_assert(offsetof(gds_mlx5_exp_send_request_t, dtype) == offsetof(gds_send_request_t, dtype), "dtype of gds_mlx5_exp_send_request_t and gds_send_request_t must be at the same offset."); + typedef struct gds_mlx5_exp_wait_request { gds_driver_type_t dtype; uint8_t pad0[4]; @@ -28,7 +39,6 @@ typedef struct gds_mlx5_exp_wait_request { struct peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; uint8_t pad1[16]; } gds_mlx5_exp_wait_request_t; - static_assert(sizeof(gds_mlx5_exp_wait_request_t) % 64 == 0, "gds_mlx5_exp_wait_request_t must be 64-byte aligned."); static_assert(sizeof(gds_mlx5_exp_wait_request_t) <= sizeof(gds_wait_request_t), "The size of gds_mlx5_exp_wait_request_t must be less than or equal to that of gds_wait_request_t."); static_assert(offsetof(gds_mlx5_exp_wait_request_t, dtype) == offsetof(gds_wait_request_t, dtype), "dtype of gds_mlx5_exp_wait_request_t and gds_wait_request_t must be at the same offset."); @@ -43,6 +53,15 @@ static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } +static inline gds_mlx5_exp_send_request_t *to_gds_mexp_send_request(gds_send_request_t *gsreq) { + assert(gsreq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + return (gds_mlx5_exp_send_request_t *)(gsreq); +} + +static inline const gds_mlx5_exp_send_request_t *to_gds_mexp_send_request(const gds_send_request_t *gsreq) { + return (const gds_mlx5_exp_send_request_t *)to_gds_mexp_send_request((const gds_send_request_t *)gsreq); +} + static inline gds_mlx5_exp_wait_request_t *to_gds_mexp_wait_request(gds_wait_request_t *gwreq) { assert(gwreq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return (gds_mlx5_exp_wait_request_t *)(gwreq); @@ -56,6 +75,10 @@ static inline uint32_t gds_mlx5_exp_get_num_wait_request_entries(gds_mlx5_exp_wa return gmexp_request->peek.entries; } +static inline uint32_t gds_mlx5_exp_get_num_send_request_entries(gds_mlx5_exp_send_request_t *gmexp_request) { + return gmexp_request->commit.entries; +} + gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, @@ -71,7 +94,12 @@ int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, - gds_send_request_t *request); + gds_mlx5_exp_send_request_t *request); + + +void gds_mlx5_exp_init_send_info(gds_mlx5_exp_send_request_t *info); +int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_mlx5_exp_send_request_t *info, gds_op_list_t &ops); +int gds_mlx5_exp_post_send_ops_on_cpu(gds_mlx5_exp_send_request_t *info, int flags = 0); void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset); void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t count); @@ -81,3 +109,5 @@ int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_req int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_mlx5_exp_wait_request_t *request, gds_op_list_t ¶ms, int flags); int gds_mlx5_exp_post_wait_descriptor(gds_mlx5_exp_wait_request_t *request, int flags); int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp_wait_request_t *request); + +int gds_mlx5_exp_rollback_qp(gds_mlx5_exp_qp_t *gmexpqp, gds_mlx5_exp_send_request_t *send_info); From 92b09f04923211e3bf6f968f369e4c4b3bc38dd6 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 04:42:33 -0400 Subject: [PATCH 19/39] Moved APIs that use gds_send_request_t to mlx5-exp.cpp --- src/apis.cpp | 58 ++++++++++++++++++------------------------------ src/gdsync.cpp | 6 +++-- src/mlx5-exp.cpp | 54 +++++++++++++++++++++++++++++++++++++++++++- src/mlx5.cpp | 3 ++- 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/src/apis.cpp b/src/apis.cpp index e9ad2ed..23577b0 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -68,12 +68,15 @@ void gds_init_ops(struct peer_op_wr *op, int count) static void gds_init_send_info(gds_send_request_t *info) { + gds_mlx5_exp_send_request_t *gmexp_info; gds_dbg("send_request=%p\n", info); memset(info, 0, sizeof(*info)); - info->commit.storage = info->wr; - info->commit.entries = sizeof(info->wr)/sizeof(info->wr[0]); - gds_init_ops(info->commit.storage, info->commit.entries); + info->dtype = GDS_DRIVER_TYPE_MLX5_EXP; + + gmexp_info = to_gds_mexp_send_request(info); + + gds_mlx5_exp_init_send_info(gmexp_info); } //----------------------------------------------------------------------------- @@ -93,37 +96,18 @@ static void gds_init_wait_request(gds_wait_request_t *request, uint32_t offset) //----------------------------------------------------------------------------- -static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t * send_info, enum ibv_exp_rollback_flags flag) +static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t *send_info) { - struct ibv_exp_rollback_ctx rollback; - int ret=0; + gds_mlx5_exp_qp_t *gmexpqp; + gds_mlx5_exp_send_request_t *gmexp_sreq; assert(qp); - assert(qp->qp); assert(send_info); - if( - flag != IBV_EXP_ROLLBACK_ABORT_UNCOMMITED && - flag != IBV_EXP_ROLLBACK_ABORT_LATE - ) - { - gds_err("erroneous ibv_exp_rollback_flags flag input value\n"); - ret=EINVAL; - goto out; - } - - /* from ibv_exp_peer_commit call */ - rollback.rollback_id = send_info->commit.rollback_id; - /* from ibv_exp_rollback_flag */ - rollback.flags = flag; - /* Reserved for future expensions, must be 0 */ - rollback.comp_mask = 0; - gds_warn("Need to rollback WQE %lx\n", rollback.rollback_id); - ret = ibv_exp_rollback_qp(qp->qp, &rollback); - if(ret) - gds_err("error %d in ibv_exp_rollback_qp\n", ret); -out: - return ret; + gmexpqp = to_gds_mexp_qp(qp); + gmexp_sreq = to_gds_mexp_send_request(send_info); + + return gds_mlx5_exp_rollback_qp(gmexpqp, gmexp_sreq); } //----------------------------------------------------------------------------- @@ -141,7 +125,7 @@ int gds_post_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) ret = gds_post_pokes_on_cpu(1, &send_info, NULL, 0); if (ret) { gds_err("error %d in gds_post_pokes_on_cpu\n", ret); - ret_roll = gds_rollback_qp(qp, &send_info, IBV_EXP_ROLLBACK_ABORT_LATE); + ret_roll = gds_rollback_qp(qp, &send_info); if (ret_roll) { gds_err("error %d in gds_rollback_qp\n", ret_roll); } @@ -180,6 +164,7 @@ int gds_prepare_send(struct gds_qp *gqp, gds_send_wr *p_ewr, { int ret = 0; gds_mlx5_exp_qp_t *gmexpqp; + gds_mlx5_exp_send_request_t *sreq; gds_init_send_info(request); assert(gqp); @@ -187,8 +172,9 @@ int gds_prepare_send(struct gds_qp *gqp, gds_send_wr *p_ewr, assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); gmexpqp = to_gds_mexp_qp(gqp); + sreq = to_gds_mexp_send_request(request); - ret = gds_mlx5_exp_prepare_send(gmexpqp, p_ewr, bad_ewr, request); + ret = gds_mlx5_exp_prepare_send(gmexpqp, p_ewr, bad_ewr, sreq); if (ret) gds_err("Error %d in gds_mlx5_exp_prepare_send.\n", ret); @@ -522,7 +508,7 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem gds_descriptor_t *desc = descs + i; switch(desc->tag) { case GDS_TAG_SEND: - n_mem_ops += desc->send->commit.entries + 2; // extra space, ugly + n_mem_ops += gds_mlx5_exp_get_num_send_request_entries(to_gds_mexp_send_request(desc->send)) + 2; // extra space, ugly break; case GDS_TAG_WAIT: n_mem_ops += gds_mlx5_exp_get_num_wait_request_entries(to_gds_mexp_wait_request(desc->wait)) + 2; // ditto @@ -584,8 +570,8 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ gds_descriptor_t *desc = descs + i; switch(desc->tag) { case GDS_TAG_SEND: { - gds_send_request_t *sreq = desc->send; - retcode = gds_post_ops(peer, sreq->commit.entries, sreq->commit.storage, params); + gds_mlx5_exp_send_request_t *sreq = to_gds_mexp_send_request(desc->send); + retcode = gds_mlx5_exp_post_send_ops(peer, sreq, params); if (retcode) { gds_err("error %d in gds_post_ops\n", retcode); ret = retcode; @@ -662,8 +648,8 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) switch(desc->tag) { case GDS_TAG_SEND: { gds_dbg("desc[%zu] SEND\n", i); - gds_send_request_t *sreq = desc->send; - retcode = gds_post_ops_on_cpu(sreq->commit.entries, sreq->commit.storage, flags); + gds_mlx5_exp_send_request_t *sreq = to_gds_mexp_send_request(desc->send); + retcode = gds_mlx5_exp_post_send_ops_on_cpu(sreq, flags); if (retcode) { gds_err("error %d in gds_post_ops_on_cpu\n", retcode); ret = retcode; diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 670bc45..9f1ddde 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -997,8 +997,9 @@ int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_ } for (int j=0; jgqp.qp, p_ewr, bad_ewr); @@ -284,6 +284,31 @@ int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, //----------------------------------------------------------------------------- +void gds_mlx5_exp_init_send_info(gds_mlx5_exp_send_request_t *info) +{ + gds_dbg("send_request=%p\n", info); + + info->commit.storage = info->wr; + info->commit.entries = sizeof(info->wr)/sizeof(info->wr[0]); + gds_init_ops(info->commit.storage, info->commit.entries); +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_mlx5_exp_send_request_t *info, gds_op_list_t &ops) +{ + return gds_post_ops(peer, info->commit.entries, info->commit.storage, ops, 0); +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_post_send_ops_on_cpu(gds_mlx5_exp_send_request_t *info, int flags) +{ + return gds_post_ops_on_cpu(info->commit.entries, info->commit.storage, flags); +} + +//----------------------------------------------------------------------------- + void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset) { gds_dbg("wait_request=%p offset=%08x\n", request, offset); @@ -562,3 +587,30 @@ int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp } return retcode; } + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_rollback_qp(gds_mlx5_exp_qp_t *gmexpqp, gds_mlx5_exp_send_request_t *send_info) +{ + struct ibv_exp_rollback_ctx rollback; + int ret = 0; + enum ibv_exp_rollback_flags flag = IBV_EXP_ROLLBACK_ABORT_LATE; + + assert(gmexpqp); + assert(gmexpqp->gqp.qp); + assert(send_info); + + /* from ibv_exp_peer_commit call */ + rollback.rollback_id = send_info->commit.rollback_id; + /* from ibv_exp_rollback_flag */ + rollback.flags = flag; + /* Reserved for future expensions, must be 0 */ + rollback.comp_mask = 0; + gds_warn("Need to rollback WQE %lx\n", rollback.rollback_id); + ret = ibv_exp_rollback_qp(gmexpqp->gqp.qp, &rollback); + if (ret) + gds_err("error %d in ibv_exp_rollback_qp\n", ret); + +out: + return ret; +} diff --git a/src/mlx5.cpp b/src/mlx5.cpp index b026374..526f544 100644 --- a/src/mlx5.cpp +++ b/src/mlx5.cpp @@ -52,9 +52,10 @@ //----------------------------------------------------------------------------- -int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *request) +int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *_request) { int retcode = 0; + const gds_mlx5_exp_send_request_t *request = to_gds_mexp_send_request(_request); size_t n_ops = request->commit.entries; peer_op_wr *op = request->commit.storage; size_t n = 0; From e088b01ef8c7c6cb95998d7ebc002103ba381d2d Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 21:46:09 -0400 Subject: [PATCH 20/39] Removed include verbs_exp.h and peer_ops.h from public header files --- include/gdsync.h | 2 -- src/gdsync.cpp | 3 +++ src/objs.hpp | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/gdsync.h b/include/gdsync.h index 7d6a45b..f2ed858 100644 --- a/include/gdsync.h +++ b/include/gdsync.h @@ -33,8 +33,6 @@ */ #include -#include -#include #include #include diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 9f1ddde..f951a54 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -33,6 +33,9 @@ #include #include +#include +#include + #include #include diff --git a/src/objs.hpp b/src/objs.hpp index 796b6bd..a57e288 100644 --- a/src/objs.hpp +++ b/src/objs.hpp @@ -27,6 +27,9 @@ #pragma once +#include +#include + static const size_t max_gpus = 16; typedef struct ibv_exp_peer_direct_attr gds_peer_attr; From 942aa8975f55c2518982dd1d69c463d29a3a360f Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 23:14:15 -0400 Subject: [PATCH 21/39] Replaced structs related to ibv_exp_ with gds_* structs --- src/gdsync.cpp | 8 +-- src/mlx5-exp.cpp | 6 +- src/objs.hpp | 165 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 171 insertions(+), 8 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index f951a54..2697dce 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1226,7 +1226,7 @@ int gds_stream_post_wait_cq_multi(CUstream stream, int count, gds_wait_request_t // If NULL returned then buffer will be allocated in system memory // by ibverbs driver. -static struct ibv_exp_peer_buf *gds_buf_alloc(ibv_exp_peer_buf_alloc_attr *attr) +static gds_peer_buf_t *gds_buf_alloc(gds_peer_buf_alloc_attr_t *attr) { assert(attr); gds_peer *peer = peer_from_id(attr->peer_id); @@ -1238,7 +1238,7 @@ static struct ibv_exp_peer_buf *gds_buf_alloc(ibv_exp_peer_buf_alloc_attr *attr) return peer->buf_alloc(peer->alloc_type, attr->length, attr->dir, attr->alignment, peer->alloc_flags); } -static int gds_buf_release(struct ibv_exp_peer_buf *pb) +static int gds_buf_release(gds_peer_buf_t *pb) { gds_dbg("freeing pb=%p\n", pb); gds_buf *buf = static_cast(pb); @@ -1247,14 +1247,14 @@ static int gds_buf_release(struct ibv_exp_peer_buf *pb) return 0; } -static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, struct ibv_exp_peer_buf *pb) +static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, gds_peer_buf_t *pb) { gds_peer *peer = peer_from_id(peer_id); gds_range *range = NULL; gds_dbg("start=%p length=%zu peer_id=%" PRIx64 " peer_buf=%p\n", start, length, peer_id, pb); - if (IBV_EXP_PEER_IOMEMORY == pb) { + if (GDS_PEER_IOMEMORY == pb) { // register as IOMEM range = peer->register_range(start, length, GDS_MEMORY_IO); } diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index dfaf91c..b75f3cb 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -53,7 +53,8 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT; attr.flags = 0; // see ibv_exp_cq_create_flags - attr.peer_direct_attrs = peer_attr; + static_assert(sizeof(gds_peer_attr) == sizeof(struct ibv_exp_peer_direct_attr)); + attr.peer_direct_attrs = (struct ibv_exp_peer_direct_attr *)(peer_attr); if (res_domain) { gds_dbg("using peer->res_domain %p for CQ\n", res_domain); attr.res_domain = res_domain; @@ -145,7 +146,8 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( exp_qp_attr.recv_cq = rx_gmexpcq->gcq.cq; exp_qp_attr.pd = pd; exp_qp_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_PEER_DIRECT; - exp_qp_attr.peer_direct_attrs = peer_attr; + static_assert(sizeof(gds_peer_attr) == sizeof(struct ibv_exp_peer_direct_attr)); + exp_qp_attr.peer_direct_attrs = (struct ibv_exp_peer_direct_attr *)peer_attr; exp_qp_attr.qp_type = qp_attr->qp_type; assert(sizeof(exp_qp_attr.cap) == sizeof(qp_attr->cap)); diff --git a/src/objs.hpp b/src/objs.hpp index a57e288..e524155 100644 --- a/src/objs.hpp +++ b/src/objs.hpp @@ -32,11 +32,172 @@ static const size_t max_gpus = 16; -typedef struct ibv_exp_peer_direct_attr gds_peer_attr; +/** + * Compatible with enum ibv_exp_peer_op + */ +enum gds_peer_op { + GDS_PEER_OP_RESERVED1 = 1, + + GDS_PEER_OP_FENCE = 0, + + GDS_PEER_OP_STORE_DWORD = 4, + GDS_PEER_OP_STORE_QWORD = 2, + GDS_PEER_OP_COPY_BLOCK = 3, + + GDS_PEER_OP_POLL_AND_DWORD = 12, + GDS_PEER_OP_POLL_NOR_DWORD = 13, + GDS_PEER_OP_POLL_GEQ_DWORD = 14, +}; + +/** + * Compatible with enum ibv_exp_peer_op_caps + */ +enum gds_peer_op_caps { + GDS_PEER_OP_FENCE_CAP = (1 << GDS_PEER_OP_FENCE), + GDS_PEER_OP_STORE_DWORD_CAP = (1 << GDS_PEER_OP_STORE_DWORD), + GDS_PEER_OP_STORE_QWORD_CAP = (1 << GDS_PEER_OP_STORE_QWORD), + GDS_PEER_OP_COPY_BLOCK_CAP = (1 << GDS_PEER_OP_COPY_BLOCK), + GDS_PEER_OP_POLL_AND_DWORD_CAP + = (1 << GDS_PEER_OP_POLL_AND_DWORD), + GDS_PEER_OP_POLL_NOR_DWORD_CAP + = (1 << GDS_PEER_OP_POLL_NOR_DWORD), + GDS_PEER_OP_POLL_GEQ_DWORD_CAP + = (1 << GDS_PEER_OP_POLL_GEQ_DWORD), +}; + + +/** + * Compatible with enum ibv_exp_peer_fence + */ +enum gds_peer_fence { + GDS_PEER_FENCE_OP_READ = (1 << 0), + GDS_PEER_FENCE_OP_WRITE = (1 << 1), + GDS_PEER_FENCE_FROM_CPU = (1 << 2), + GDS_PEER_FENCE_FROM_HCA = (1 << 3), + GDS_PEER_FENCE_MEM_SYS = (1 << 4), + GDS_PEER_FENCE_MEM_PEER = (1 << 5), +}; + +/** + * Indicate HW entities supposed to access memory buffer: + * GDS_PEER_DIRECTION_FROM_X means X writes to the buffer + * GDS_PEER_DIRECTION_TO_Y means Y read from the buffer + * + * Compatible with enum ibv_exp_peer_direction + */ +enum gds_peer_direction { + GDS_PEER_DIRECTION_FROM_CPU = (1 << 0), + GDS_PEER_DIRECTION_FROM_HCA = (1 << 1), + GDS_PEER_DIRECTION_FROM_PEER = (1 << 2), + GDS_PEER_DIRECTION_TO_CPU = (1 << 3), + GDS_PEER_DIRECTION_TO_HCA = (1 << 4), + GDS_PEER_DIRECTION_TO_PEER = (1 << 5), +}; + +/** + * Compatible with enum ibv_exp_peer_direct_attr_mask + */ +enum gds_peer_direct_attr_mask { + GDS_PEER_DIRECT_VERSION = (1 << 0) /* Must be set */ +}; + +/** + * Compatible with IBV_EXP_PEER_IOMEMORY + */ +#define GDS_PEER_IOMEMORY ((struct gds_buf *)-1UL) + +/** + * Compatible with struct ibv_exp_peer_buf_alloc_attr + */ +typedef struct gds_peer_buf_alloc_attr { + size_t length; + /* Bitmask from enum gds_peer_direction */ + uint32_t dir; + /* The ID of the peer device which will be + * * accessing the allocated buffer + * */ + uint64_t peer_id; + /* Data alignment */ + uint32_t alignment; + /* Reserved for future extensions, must be 0 */ + uint32_t comp_mask; +} gds_peer_buf_alloc_attr_t; + + +/** + * Compatible with struct ibv_exp_peer_buf + */ +typedef struct gds_peer_buf { + void *addr; + size_t length; + /* Reserved for future extensions, must be 0 */ + uint32_t comp_mask; +} gds_peer_buf_t; + +/** + * Compatible with struct ibv_exp_peer_direct_attr + */ +typedef struct { + /* Unique ID per peer device. + * Used to identify specific HW devices where relevant. + */ + uint64_t peer_id; + /* buf_alloc callback should return gds_peer_buf_t with buffer + * of at least attr->length. + * @attr: description of desired buffer + * + * Buffer should be mapped in the application address space + * for read/write (depends on attr->dir value). + * attr->dir value is supposed to indicate the expected directions + * of access to the buffer, to allow optimization by the peer driver. + * If NULL returned then buffer will be allocated in system memory + * by ibverbs driver. + */ + gds_peer_buf_t *(*buf_alloc)(gds_peer_buf_alloc_attr_t *attr); + /* If buffer was allocated by buf_alloc then buf_release will be + * called to release it. + * @pb: struct returned by buf_alloc + * + * buf_release is responsible to release everything allocated by + * buf_alloc. + * Return 0 on succes. + */ + int (*buf_release)(gds_peer_buf_t *pb); + /* register_va callback should register virtual address from the + * application as an area the peer is allowed to access. + * @start: pointer to beginning of region in virtual space + * @length: length of region + * @peer_id: the ID of the peer device which will be accessing + * the region. + * @pb: if registering a buffer that was returned from buf_alloc(), + * pb is the struct that was returned. If registering io memory area, + * pb is GDS_PEER_IOMEMORY. Otherwise - NULL + * + * Return id of registered address on success, 0 on failure. + */ + uint64_t (*register_va)(void *start, size_t length, uint64_t peer_id, + gds_peer_buf_t *pb); + /* If virtual address was registered with register_va then + * unregister_va will be called to unregister it. + * @target_id: id returned by register_va + * @peer_id: the ID of the peer device passed to register_va + * + * Return 0 on success. + */ + int (*unregister_va)(uint64_t target_id, uint64_t peer_id); + /* Bitmask from gds_peer_op_caps */ + uint64_t caps; + /* Maximal length of DMA operation the peer can do in copy-block */ + size_t peer_dma_op_map_len; + /* From gds_peer_direct_attr_mask */ + uint32_t comp_mask; + /* Feature version, must be 1 */ + uint32_t version; +} gds_peer_attr; struct gds_peer; -struct gds_buf: ibv_exp_peer_buf { +struct gds_buf: gds_peer_buf_t { gds_peer *peer; CUdeviceptr peer_addr; void *handle; From 35db8665b4c485fdf2de4ddd7db2796c97664abd Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 23:19:54 -0400 Subject: [PATCH 22/39] Removed include verbs_exp.h and peer_ops.h from objs.hpp --- src/mlx5-exp.hpp | 1 + src/objs.cpp | 8 ++++---- src/objs.hpp | 3 --- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index 7e4ba14..cb43147 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/objs.cpp b/src/objs.cpp index 1dac250..9c9f6ff 100644 --- a/src/objs.cpp +++ b/src/objs.cpp @@ -71,7 +71,7 @@ gds_buf *gds_peer::buf_alloc_cq(size_t length, uint32_t dir, uint32_t alignment, { gds_buf *buf = NULL; switch (dir) { - case (IBV_EXP_PEER_DIRECTION_FROM_HCA|IBV_EXP_PEER_DIRECTION_TO_PEER|IBV_EXP_PEER_DIRECTION_TO_CPU): + case (GDS_PEER_DIRECTION_FROM_HCA|GDS_PEER_DIRECTION_TO_PEER|GDS_PEER_DIRECTION_TO_CPU): // CQ buf if (GDS_ALLOC_CQ_ON_GPU == (flags & GDS_ALLOC_CQ_MASK)) { gds_dbg("allocating CQ on GPU mem\n"); @@ -80,14 +80,14 @@ gds_buf *gds_peer::buf_alloc_cq(size_t length, uint32_t dir, uint32_t alignment, gds_dbg("allocating CQ on Host mem\n"); } break; - case (IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_CPU): + case (GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_CPU): // CQ peer buf, helper buffer // on SYSMEM for the near future // GPU does a store to the 'busy' field as part of the peek_cq task // CPU polls on that field gds_dbg("allocating CQ peer buf on Host mem\n"); break; - case (IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_HCA): + case (GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_HCA): gds_dbg("allocating CQ dbrec on Host mem\n"); break; default: @@ -101,7 +101,7 @@ gds_buf *gds_peer::buf_alloc_wq(size_t length, uint32_t dir, uint32_t alignment, { gds_buf *buf = NULL; switch (dir) { - case IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_HCA: + case GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_HCA: // dbrec if (GDS_ALLOC_DBREC_ON_GPU == (flags & GDS_ALLOC_DBREC_MASK)) { gds_dbg("allocating DBREC on GPU mem\n"); diff --git a/src/objs.hpp b/src/objs.hpp index e524155..71f4ea4 100644 --- a/src/objs.hpp +++ b/src/objs.hpp @@ -27,9 +27,6 @@ #pragma once -#include -#include - static const size_t max_gpus = 16; /** From 24a0a20abb727f3c07491055b8154018427abfe0 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 9 Sep 2021 23:20:52 -0400 Subject: [PATCH 23/39] Removed include verbs_exp.h from objs.cpp --- src/objs.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/objs.cpp b/src/objs.cpp index 9c9f6ff..475c9d3 100644 --- a/src/objs.cpp +++ b/src/objs.cpp @@ -39,7 +39,6 @@ using namespace std; #include -#include #include #include "gdsync.h" From 5d7c328a1240b3e81d9e632f699b9f310a4f880f Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:17:58 -0400 Subject: [PATCH 24/39] Removed include verbs_exp.h and peer_ops.h from gdsync.cpp --- src/gdsync.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 2697dce..86d277d 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -33,9 +33,6 @@ #include #include -#include -#include - #include #include From bf1a35807915f938e22d2c8acb95d2a3bf505cb9 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:19:50 -0400 Subject: [PATCH 25/39] Relaced all IBV_EXP_ enum with GDS_ in gdsync.cpp --- src/gdsync.cpp | 86 +++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 86d277d..1af7ed6 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -783,13 +783,13 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis //int flags = 0; gds_dbg("op[%zu] type:%08x\n", n, op->type); switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { + case GDS_PEER_OP_FENCE: { gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + if (fence_op == GDS_PEER_FENCE_OP_READ) { gds_dbg("nothing to do for read fences\n"); //retcode = EINVAL; break; @@ -808,17 +808,17 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis //retcode = 0; } else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { gds_err("unexpected from fence\n"); retcode = EINVAL; break; } int flags = 0; - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { gds_dbg("using light membar\n"); flags = GDS_MEMBAR_DEFAULT | GDS_MEMBAR_MLX5; } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { gds_dbg("using heavy membar\n"); flags = GDS_MEMBAR_SYS | GDS_MEMBAR_MLX5; } @@ -832,7 +832,7 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis } break; } - case IBV_EXP_PEER_OP_STORE_DWORD: { + case GDS_PEER_OP_STORE_DWORD: { CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + op->wr.dword_va.offset; uint32_t data = op->wr.dword_va.data; @@ -867,7 +867,7 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis } break; } - case IBV_EXP_PEER_OP_STORE_QWORD: { + case GDS_PEER_OP_STORE_QWORD: { CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + op->wr.qword_va.offset; uint64_t data = op->wr.qword_va.data; @@ -910,7 +910,7 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis break; } - case IBV_EXP_PEER_OP_COPY_BLOCK: { + case GDS_PEER_OP_COPY_BLOCK: { CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + op->wr.copy_op.offset; size_t len = op->wr.copy_op.len; @@ -931,9 +931,9 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis retcode = gds_fill_inlcpy(peer, ops, dev_ptr, src, len, flags); break; } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { int poll_cond; CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + op->wr.dword_va.offset; @@ -946,13 +946,13 @@ int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_lis gds_dbg("OP_WAIT_DWORD dev_ptr=%llx data=%" PRIx32 " type=%" PRIx32 "\n", dev_ptr, data, (uint32_t)op->type); switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: poll_cond = GDS_WAIT_COND_NOR; break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: poll_cond = GDS_WAIT_COND_GEQ; break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_AND_DWORD: poll_cond = GDS_WAIT_COND_AND; break; default: @@ -1036,28 +1036,28 @@ int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) gds_dbg("op[%zu]=%p\n", n, op); //gds_dbg("op[%zu]=%p type:%08x\n", n, op, op->type); switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { + case GDS_PEER_OP_FENCE: { gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + if (fence_op == GDS_PEER_FENCE_OP_READ) { gds_warnc(1, "nothing to do for read fences\n"); //retcode = EINVAL; break; } else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); retcode = EINVAL; break; } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { gds_dbg("using light membar\n"); wmb(); } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { gds_dbg("using heavy membar\n"); wmb(); } @@ -1069,7 +1069,7 @@ int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) } break; } - case IBV_EXP_PEER_OP_STORE_DWORD: { + case GDS_PEER_OP_STORE_DWORD: { uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); uint32_t data = op->wr.dword_va.data; // A || B || C || E @@ -1077,14 +1077,14 @@ int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) gds_atomic_set(ptr, data); break; } - case IBV_EXP_PEER_OP_STORE_QWORD: { + case GDS_PEER_OP_STORE_QWORD: { uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); uint64_t data = op->wr.qword_va.data; gds_dbg("STORE_QWORD ptr=%p data=%016" PRIx64 "\n", ptr, data); gds_atomic_set(ptr, data); break; } - case IBV_EXP_PEER_OP_COPY_BLOCK: { + case GDS_PEER_OP_COPY_BLOCK: { uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); uint64_t *src = (uint64_t*)op->wr.copy_op.src; size_t n_bytes = op->wr.copy_op.len; @@ -1092,9 +1092,9 @@ int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) gds_bf_copy(ptr, src, n_bytes); break; } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { int poll_cond; uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); uint32_t value = op->wr.dword_va.data; @@ -1106,13 +1106,13 @@ int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) do { uint32_t data = gds_atomic_get(ptr); switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: done = (0 != ~(data | value)); break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: done = ((int32_t)data - (int32_t)value >= 0); break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_AND_DWORD: done = (0 != (data & value)); break; default: @@ -1453,28 +1453,28 @@ static void gds_init_peer(gds_peer *peer, CUdevice dev, int gpu_id) peer->attr.register_va = gds_register_va; peer->attr.unregister_va = gds_unregister_va; - peer->attr.caps = ( IBV_EXP_PEER_OP_STORE_DWORD_CAP | - IBV_EXP_PEER_OP_STORE_QWORD_CAP | - IBV_EXP_PEER_OP_FENCE_CAP | - IBV_EXP_PEER_OP_POLL_AND_DWORD_CAP ); + peer->attr.caps = ( GDS_PEER_OP_STORE_DWORD_CAP | + GDS_PEER_OP_STORE_QWORD_CAP | + GDS_PEER_OP_FENCE_CAP | + GDS_PEER_OP_POLL_AND_DWORD_CAP ); if (peer->has_wait_nor) { gds_dbg("enabling NOR feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_POLL_NOR_DWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_POLL_NOR_DWORD_CAP; } else - peer->attr.caps |= IBV_EXP_PEER_OP_POLL_GEQ_DWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_POLL_GEQ_DWORD_CAP; if (peer->has_inlcpy) { gds_dbg("enabling COPY BLOCK feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_COPY_BLOCK_CAP; + peer->attr.caps |= GDS_PEER_OP_COPY_BLOCK_CAP; } else if (peer->has_write64 || gds_simulate_write64()) { gds_dbg("enabling STORE QWORD feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_STORE_QWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_STORE_QWORD_CAP; } gds_dbg("caps=%016lx\n", peer->attr.caps); peer->attr.peer_dma_op_map_len = GDS_GPU_MAX_INLINE_SIZE; - peer->attr.comp_mask = IBV_EXP_PEER_DIRECT_VERSION; + peer->attr.comp_mask = GDS_PEER_DIRECT_VERSION; peer->attr.version = 1; peer->tq = new task_queue; From f95c72d9ecf982f2e4714ee0ec1eb10356bc7513 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:28:53 -0400 Subject: [PATCH 26/39] Replaced all IBV_EXP_ enum with GDS_ in mlx5.cpp --- src/mlx5.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mlx5.cpp b/src/mlx5.cpp index 526f544..c67a2f6 100644 --- a/src/mlx5.cpp +++ b/src/mlx5.cpp @@ -64,25 +64,25 @@ int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request for (; op && n < n_ops; op = op->next, ++n) { switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { + case GDS_PEER_OP_FENCE: { gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + if (fence_op == GDS_PEER_FENCE_OP_READ) { gds_dbg("nothing to do for read fences\n"); break; } - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { gds_err("unexpected from fence\n"); retcode = EINVAL; break; } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { gds_dbg("using light membar\n"); mlx5_i->membar = 1; } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { gds_dbg("using heavy membar\n"); mlx5_i->membar_full = 1; } @@ -93,7 +93,7 @@ int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request } break; } - case IBV_EXP_PEER_OP_STORE_DWORD: { + case GDS_PEER_OP_STORE_DWORD: { CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + op->wr.dword_va.offset; uint32_t data = op->wr.dword_va.data; @@ -107,7 +107,7 @@ int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request mlx5_i->dbrec_value = data; break; } - case IBV_EXP_PEER_OP_STORE_QWORD: { + case GDS_PEER_OP_STORE_QWORD: { CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + op->wr.qword_va.offset; uint64_t data = op->wr.qword_va.data; @@ -121,7 +121,7 @@ int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request mlx5_i->db_value = data; break; } - case IBV_EXP_PEER_OP_COPY_BLOCK: { + case GDS_PEER_OP_COPY_BLOCK: { CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + op->wr.copy_op.offset; size_t len = op->wr.copy_op.len; @@ -136,9 +136,9 @@ int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request mlx5_i->db_value = *(uint64_t*)src; break; } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { gds_err("unexpected polling op in send request\n"); retcode = EINVAL; break; From 6a23931a2dcdf5c321eb7705f937bba22be7502b Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:31:04 -0400 Subject: [PATCH 27/39] Relaced all IBV_EXP_ enum with GDS_ enum in gdsync_debug_hostregister_bug.cpp --- src/gdsync_debug_hostregister_bug.cpp | 56 +++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/gdsync_debug_hostregister_bug.cpp b/src/gdsync_debug_hostregister_bug.cpp index 1e36d08..63d0f3b 100644 --- a/src/gdsync_debug_hostregister_bug.cpp +++ b/src/gdsync_debug_hostregister_bug.cpp @@ -704,11 +704,11 @@ static int gds_post_ops(size_t n_ops, struct peer_op_wr *op, CUstreamBatchMemOpP switch(op->type) { case IBV_PEER_OP_FENCE: { gds_dbg("OP_FENCE: fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + if (fence_op == GDS_PEER_FENCE_OP_READ) { gds_dbg("nothing to do for read fences\n"); //retcode = EINVAL; break; @@ -727,17 +727,17 @@ static int gds_post_ops(size_t n_ops, struct peer_op_wr *op, CUstreamBatchMemOpP //retcode = 0; } else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { gds_err("unexpected from fence\n"); retcode = EINVAL; break; } int flags = 0; - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { gds_dbg("using light membar\n"); flags = GDS_MEMBAR_DEFAULT; } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { gds_dbg("using heavy membar\n"); flags = GDS_MEMBAR_SYS; } @@ -973,26 +973,26 @@ static int gds_post_ops_on_cpu(size_t n_descs, struct peer_op_wr *op) switch(op->type) { case IBV_PEER_OP_FENCE: { gds_dbg("fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + if (fence_op == GDS_PEER_FENCE_OP_READ) { gds_warnc(1, "nothing to do for read fences\n"); //retcode = EINVAL; break; } else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); retcode = EINVAL; break; } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { gds_dbg("using light membar\n"); wmb(); } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { gds_dbg("using heavy membar\n"); wmb(); } @@ -1322,7 +1322,7 @@ static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, st gds_dbg("start=%p length=%zu peer_id=%"PRIx64" peer_buf=%p\n", start, length, peer_id, pb); - if (IBV_EXP_PEER_IOMEMORY == pb) { + if (GDS_PEER_IOMEMORY == pb) { // register as IOMEM range = peer->register_range(start, length, GDS_MEMORY_IO); } @@ -1375,25 +1375,25 @@ static void gds_init_peer_attr(gds_peer_attr *attr, gds_peer *peer) attr->register_va = gds_register_va; attr->unregister_va = gds_unregister_va; - attr->caps = ( IBV_EXP_PEER_OP_STORE_DWORD_CAP | - IBV_EXP_PEER_OP_STORE_QWORD_CAP | - IBV_EXP_PEER_OP_FENCE_CAP | - IBV_EXP_PEER_OP_POLL_AND_DWORD_CAP ); + attr->caps = ( GDS_PEER_OP_STORE_DWORD_CAP | + GDS_PEER_OP_STORE_QWORD_CAP | + GDS_PEER_OP_FENCE_CAP | + GDS_PEER_OP_POLL_AND_DWORD_CAP ); if (gpu_does_support_nor(peer)) - attr->caps |= IBV_EXP_PEER_OP_POLL_NOR_DWORD_CAP; + attr->caps |= GDS_PEER_OP_POLL_NOR_DWORD_CAP; else - attr->caps |= IBV_EXP_PEER_OP_POLL_GEQ_DWORD_CAP; + attr->caps |= GDS_PEER_OP_POLL_GEQ_DWORD_CAP; if (gds_enable_inlcpy()) { - attr->caps |= IBV_EXP_PEER_OP_COPY_BLOCK_CAP; + attr->caps |= GDS_PEER_OP_COPY_BLOCK_CAP; } else if (gds_enable_write64() || gds_simulate_write64()) { - attr->caps |= IBV_EXP_PEER_OP_STORE_QWORD_CAP; + attr->caps |= GDS_PEER_OP_STORE_QWORD_CAP; } gds_dbg("caps=%016lx\n", attr->caps); attr->peer_dma_op_map_len = GDS_GPU_MAX_INLINE_SIZE; - attr->comp_mask = IBV_EXP_PEER_DIRECT_VERSION; + attr->comp_mask = GDS_PEER_DIRECT_VERSION; attr->version = 1; gds_dbg("peer_attr: peer_id=%"PRIx64"\n", attr->peer_id); @@ -1536,13 +1536,13 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds // the CQE without updating the tracking variables if (flags & GDS_CREATE_QP_GPU_INVALIDATE_RX_CQ) { gds_warn("IGNORE_RQ_OVERFLOW\n"); - qp_attr->exp_create_flags |= IBV_EXP_QP_CREATE_IGNORE_RQ_OVERFLOW; - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS; + qp_attr->exp_create_flags |= GDS_QP_CREATE_IGNORE_RQ_OVERFLOW; + qp_attr->comp_mask |= GDS_QP_INIT_ATTR_CREATE_FLAGS; } if (flags & GDS_CREATE_QP_GPU_INVALIDATE_TX_CQ) { gds_warn("IGNORE_SQ_OVERFLOW\n"); - qp_attr->exp_create_flags |= IBV_EXP_QP_CREATE_IGNORE_SQ_OVERFLOW; - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS; + qp_attr->exp_create_flags |= GDS_QP_CREATE_IGNORE_SQ_OVERFLOW; + qp_attr->comp_mask |= GDS_QP_INIT_ATTR_CREATE_FLAGS; } gds_dbg("before gds_register_peer_ex\n"); From ab745dea2d5772c9a12fabc6587bfed85832985f Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:37:10 -0400 Subject: [PATCH 28/39] Fixed bugs in gds_dump_wait_request --- src/gdsync.cpp | 12 +++++++----- src/mlx5-exp.cpp | 14 ++++++-------- src/mlx5-exp.hpp | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 1af7ed6..b01cd36 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1173,12 +1173,14 @@ int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uin void gds_dump_wait_request(gds_wait_request_t *request, size_t count) { - gds_mlx5_exp_wait_request_t *gmexp_request; - if (count == 0) - return; + for (size_t j = 0; j < count; ++j) { + gds_mlx5_exp_wait_request_t *gmexp_request; + if (count == 0) + return; - gmexp_request = to_gds_mexp_wait_request(request); - gds_mlx5_exp_dump_wait_request(gmexp_request, count); + gmexp_request = to_gds_mexp_wait_request(&request[j]); + gds_mlx5_exp_dump_wait_request(gmexp_request, j); + } } //----------------------------------------------------------------------------- diff --git a/src/mlx5-exp.cpp b/src/mlx5-exp.cpp index b75f3cb..7806438 100644 --- a/src/mlx5-exp.cpp +++ b/src/mlx5-exp.cpp @@ -381,15 +381,13 @@ static void gds_mlx5_exp_dump_ops(struct peer_op_wr *op, size_t count) //----------------------------------------------------------------------------- -void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t count) +void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t idx) { - for (size_t j = 0; j < count; ++j) { - struct ibv_exp_peer_peek *peek = &request[j].peek; - gds_dbg("req[%zu] entries:%u whence:%u offset:%u peek_id:%" PRIx64 " comp_mask:%08x\n", - j, peek->entries, peek->whence, peek->offset, - peek->peek_id, peek->comp_mask); - gds_mlx5_exp_dump_ops(peek->storage, peek->entries); - } + struct ibv_exp_peer_peek *peek = &request->peek; + gds_dbg("req[%zu] entries:%u whence:%u offset:%u peek_id:%" PRIx64 " comp_mask:%08x\n", + idx, peek->entries, peek->whence, peek->offset, + peek->peek_id, peek->comp_mask); + gds_mlx5_exp_dump_ops(peek->storage, peek->entries); } //----------------------------------------------------------------------------- diff --git a/src/mlx5-exp.hpp b/src/mlx5-exp.hpp index cb43147..1d67137 100644 --- a/src/mlx5-exp.hpp +++ b/src/mlx5-exp.hpp @@ -103,7 +103,7 @@ int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_mlx5_exp_send_request_t *info int gds_mlx5_exp_post_send_ops_on_cpu(gds_mlx5_exp_send_request_t *info, int flags = 0); void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset); -void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t count); +void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t idx); int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_request_t *request, int flags); int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t *dw, uint32_t val); int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_request_t *request); From f43f905e8b4dfa2d7fcb027ab02cbdc1b0a2a2c0 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:43:44 -0400 Subject: [PATCH 29/39] Updated some functions in gdsync_debug_hostregister_bug.cpp to reflect changes in gdsync.cpp --- src/gdsync_debug_hostregister_bug.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/gdsync_debug_hostregister_bug.cpp b/src/gdsync_debug_hostregister_bug.cpp index 63d0f3b..2537a74 100644 --- a/src/gdsync_debug_hostregister_bug.cpp +++ b/src/gdsync_debug_hostregister_bug.cpp @@ -41,6 +41,7 @@ #include "objs.hpp" #include "archutils.h" #include "mlnxutils.h" +#include "mlx5-exp.hpp" //----------------------------------------------------------------------------- @@ -1233,12 +1234,13 @@ static void gds_dump_ops(struct peer_op_wr *op, size_t count) void gds_dump_wait_request(gds_wait_request_t *request, size_t count) { - for (size_t j=0; jentries, peek->whence, peek->offset, - peek->peek_id, peek->comp_mask); - gds_dump_ops(peek->storage, peek->entries); + for (size_t j = 0; j < count; ++j) { + gds_mlx5_exp_wait_request_t *gmexp_request; + if (count == 0) + return; + + gmexp_request = to_gds_mexp_wait_request(&request[j]); + gds_mlx5_exp_dump_wait_request(gmexp_request, j); } } @@ -1315,7 +1317,7 @@ static int gds_buf_release(struct ibv_peer_buf *pb) return 0; } -static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, struct ibv_exp_peer_buf *pb) +static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, gds_peer_buf_t *pb) { gds_peer *peer = peer_from_id(peer_id); gds_range *range = NULL; From aeb1ed7da448b826d83a8b7dd71877f213058c83 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 01:52:03 -0400 Subject: [PATCH 30/39] Removed include verbs_exp.h and peer_ops.h from all files except from mlx-exp.* --- src/apis.cpp | 1 - src/mem.cpp | 1 - src/memmgr.cpp | 1 - tests/gds_poll_lat.c | 1 - tests/gds_sanity.cpp | 1 - tests/gpu.cpp | 1 - tests/gpu.h | 1 - tests/pingpong.h | 1 - 8 files changed, 8 deletions(-) diff --git a/src/apis.cpp b/src/apis.cpp index 23577b0..69e491c 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -40,7 +40,6 @@ //using namespace std; //#include -//#include //#include #include "gdsync.h" diff --git a/src/mem.cpp b/src/mem.cpp index 59a6af6..7cf3602 100644 --- a/src/mem.cpp +++ b/src/mem.cpp @@ -40,7 +40,6 @@ using namespace std; #include -#include #include #include "gdsync.h" diff --git a/src/memmgr.cpp b/src/memmgr.cpp index ab3e490..6d6a21c 100644 --- a/src/memmgr.cpp +++ b/src/memmgr.cpp @@ -40,7 +40,6 @@ using namespace std; #include -#include #include #include "gdsync.h" diff --git a/tests/gds_poll_lat.c b/tests/gds_poll_lat.c index cf2147b..17c4fdb 100644 --- a/tests/gds_poll_lat.c +++ b/tests/gds_poll_lat.c @@ -11,7 +11,6 @@ #include #include -#include #include #include #include diff --git a/tests/gds_sanity.cpp b/tests/gds_sanity.cpp index 910032b..5394833 100644 --- a/tests/gds_sanity.cpp +++ b/tests/gds_sanity.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include #include diff --git a/tests/gpu.cpp b/tests/gpu.cpp index 6d7da67..8bee5e3 100644 --- a/tests/gpu.cpp +++ b/tests/gpu.cpp @@ -29,7 +29,6 @@ #include #include -#include #include "gdrapi.h" #include "gdsync.h" diff --git a/tests/gpu.h b/tests/gpu.h index 401e88f..0fde885 100644 --- a/tests/gpu.h +++ b/tests/gpu.h @@ -28,7 +28,6 @@ #pragma once #include -#include #ifdef USE_PROFILE diff --git a/tests/pingpong.h b/tests/pingpong.h index 32f020b..9cdc03e 100644 --- a/tests/pingpong.h +++ b/tests/pingpong.h @@ -34,7 +34,6 @@ #define IBV_PINGPONG_H #include -#include enum ibv_mtu pp_mtu_to_enum(int mtu); uint16_t pp_get_local_lid(struct ibv_context *context, int port); From 59d20f5ef2ae8d6b7c4b5debad472fe28a55bfe9 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 00:36:27 -0700 Subject: [PATCH 31/39] Updated configure.ac and Makefile.am to make exp-verbs optional --- Makefile.am | 9 +++++++-- configure.ac | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile.am b/Makefile.am index 27fce57..84a8085 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,10 +23,15 @@ libgdsyncincludedir = $(includedir)/gdsync libgdsyncinclude_HEADERS = include/gdsync/core.h include/gdsync/device.cuh include/gdsync/mlx5.h include/gdsync/tools.h src_libgdsync_la_CFLAGS = $(AM_CFLAGS) -src_libgdsync_la_SOURCES = src/gdsync.cpp src/memmgr.cpp src/mem.cpp src/objs.cpp src/apis.cpp src/mlx5.cpp src/mlx5-exp.cpp include/gdsync.h +src_libgdsync_la_SOURCES = src/gdsync.cpp src/memmgr.cpp src/mem.cpp src/objs.cpp src/apis.cpp src/mlx5.cpp include/gdsync.h src_libgdsync_la_LDFLAGS = -version-info @VERSION_INFO@ -noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h src/mlx5-exp.hpp +noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h + +if COMPILE_EXP_VERBS +src_libgdsync_la_SOURCES += src/mlx5-exp.cpp +noinst_HEADERS += src/mlx5-exp.hpp +endif # if enabled at configure time diff --git a/configure.ac b/configure.ac index e20f313..dab8d6a 100644 --- a/configure.ac +++ b/configure.ac @@ -190,11 +190,20 @@ dnl Checks for Verbs support AC_CHECK_LIB(ibverbs, ibv_get_device_list, [], AC_MSG_ERROR([ibv_get_device_list() not found. libgdsync requires libibverbs.])) -AC_CHECK_LIB(ibverbs, ibv_exp_create_qp, - AC_MSG_ERROR([ibv_exp_create_qp not found. libgdsync requires verbs extension support.])) +AC_CHECK_LIB(ibverbs, ibv_exp_create_qp, [have_exp_verbs=1]) +AC_CHECK_HEADER([infiniband/peer_ops.h], [have_peer_ops=1], [], +[[ +#include +]]) + +if test "x$have_exp_verbs" != "x" && test "x$have_peer_ops" != "x"; then + AC_DEFINE([HAVE_EXP_VERBS], [1], [Define if exp-verbs exists.]) + $enable_exp_verbs=1 +else + AC_MSG_WARN([This version of libgdsync cannot be used with out exp-verbs.]) +fi +AM_CONDITIONAL([COMPILE_EXP_VERBS], [test "x$enable_exp_verbs" != "x"]) -AC_CHECK_HEADER(infiniband/peer_ops.h, [], - AC_MSG_ERROR([ not found. libgdsync requires verbs peer-direct support.])) AC_HEADER_STDC dnl Checks for typedefs, structures, and compiler characteristics. From bfb683bd2f8e5e21e462999d2e7a115cf46fdb8f Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 10 Sep 2021 03:58:09 -0400 Subject: [PATCH 32/39] Fixed bug in configure.ac related to checking for exp-verbs --- configure.ac | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index dab8d6a..34bc650 100644 --- a/configure.ac +++ b/configure.ac @@ -190,7 +190,8 @@ dnl Checks for Verbs support AC_CHECK_LIB(ibverbs, ibv_get_device_list, [], AC_MSG_ERROR([ibv_get_device_list() not found. libgdsync requires libibverbs.])) -AC_CHECK_LIB(ibverbs, ibv_exp_create_qp, [have_exp_verbs=1]) +dnl ibv_exp_create_qp is an inline function. So, we check for exp_cmd instead. +AC_CHECK_LIB(ibverbs, ibv_exp_cmd_create_qp, [have_exp_verbs=1]) AC_CHECK_HEADER([infiniband/peer_ops.h], [have_peer_ops=1], [], [[ #include @@ -198,7 +199,7 @@ AC_CHECK_HEADER([infiniband/peer_ops.h], [have_peer_ops=1], [], if test "x$have_exp_verbs" != "x" && test "x$have_peer_ops" != "x"; then AC_DEFINE([HAVE_EXP_VERBS], [1], [Define if exp-verbs exists.]) - $enable_exp_verbs=1 + enable_exp_verbs=1 else AC_MSG_WARN([This version of libgdsync cannot be used with out exp-verbs.]) fi From e61a4057921fed209d9082906600774b6266ee1e Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 14 Oct 2021 23:08:59 -0400 Subject: [PATCH 33/39] Defined gds_transport_t --- src/transport.hpp | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/transport.hpp diff --git a/src/transport.hpp b/src/transport.hpp new file mode 100644 index 0000000..ddef072 --- /dev/null +++ b/src/transport.hpp @@ -0,0 +1,63 @@ +/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +typedef struct gds_transport { + int (*create_qp)(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, gds_peer *peer, gds_peer_attr *peer_attr, int flags, gds_qp_t **gqp); + int (*destroy_qp)(gds_qp_t *gqp); + int (*rollback_qp)(gds_qp_t *gqp, gds_send_request_t *request); + + void (*init_send_info)(gds_send_request_t *request); + int (*post_send_ops)(gds_peer *peer, gds_send_request_t *request, gds_op_list_t &ops); + int (*post_send_ops_on_cpu)(gds_send_request_t *request, int flags = 0); + int (*prepare_send)(gds_qp_t *gqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request); + uint32_t (*get_num_send_request_entries)(gds_send_request_t *request); + + void (*init_wait_request)(gds_wait_request_t *request, uint32_t offset); + void (*dump_wait_request)(gds_wait_request_t *request, size_t idx); + int (*stream_post_wait_descriptor)(gds_peer *peer, gds_wait_request_t *request, gds_op_list_t ¶ms, int flags); + int (*post_wait_descriptor)(gds_wait_request_t *request, int flags); + int (*get_wait_descs)(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *request); + uint32_t (*get_num_wait_request_entries)(gds_wait_request_t *request); + + int (*prepare_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request, in flags); + int (*append_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request, in flags); + int (*abort_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request); +} gds_transport_t; + +extern gds_transport_t *gds_main_transport; + +int gds_transport_mlx5_exp_init(gds_transport_t *transport); + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * tab-width: 8 + * End: + */ From 8720c8666cbfe40d4ace8c15095d0b30c9c6be86 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Thu, 14 Oct 2021 23:09:45 -0400 Subject: [PATCH 34/39] Moved mlx5-exp.* into transports/mlx5-exp and implemented gds_transport_mlx5_exp_init --- src/transports/mlx5-exp/.mlx5-exp.cpp.swp | Bin 0 -> 40960 bytes src/transports/mlx5-exp/.mlx5-exp.hpp.swp | Bin 0 -> 16384 bytes src/{ => transports/mlx5-exp}/mlx5-exp.cpp | 30 +++++++++++++++++++++ src/{ => transports/mlx5-exp}/mlx5-exp.hpp | 0 4 files changed, 30 insertions(+) create mode 100644 src/transports/mlx5-exp/.mlx5-exp.cpp.swp create mode 100644 src/transports/mlx5-exp/.mlx5-exp.hpp.swp rename src/{ => transports/mlx5-exp}/mlx5-exp.cpp (94%) rename src/{ => transports/mlx5-exp}/mlx5-exp.hpp (100%) diff --git a/src/transports/mlx5-exp/.mlx5-exp.cpp.swp b/src/transports/mlx5-exp/.mlx5-exp.cpp.swp new file mode 100644 index 0000000000000000000000000000000000000000..09e0b7542379f4809868a318355bb10c5039cc58 GIT binary patch literal 40960 zcmeI53v^{wd4R8#hoihKJ}cHkV8&#~WM<@LGh`r40z_t#OeVvCikF*vPm;?d_ukxl zGs!SKl-kuUDX2?%)S^;R)CDdpw4zAWiqfhT-;XL#T3^*lwUr`>O22=f{W|BKo5wga zW!amxzMSNoefHVs-_QTw|K4Y8-FX9Bf|qx#%kXnlCUfo2CUZBQ_^D_A(U&rr%5<)} zvpzXHCf64DnJP~dOY0X1pf6o+BV0RNEnghw>$SC0#j%M(ZFecZrm$;mt(sq3ujWd% zO1a9l>8bgX*M##G`$t#4Qd#^ltq?5c_Pm8yx-n+fzfePPQb_ zl0ZuWEeW(F(2_t)0xb!&B+!z;@1_Lmm1ktG;MIOlzuuwx__W0FApL)+kIzV4H~$y) z`KiS96^Zj-Ph3A(|G!xudvyX#Iu6n2C-w2_#C4PIv-I`bkNuVWxmIQt$C6Fs*GHZ^^WI~Ynf1^kLg#Yh@ z+u&As3tSEoZWSC4hrvTfWHR@_N8uxICG^22`1RqL%qQUEkcB>Y2K?qQuED>-L+~%~ zMz{v@a56mptW4%@uoH@~5ncp`z`^j7XJ#_rfRDjlFb&(`cz6~(4Knae3=e+;cfdUK z!oOpzxE?NpQ{XW80x$kfxEB5#reOvA2*bmhVIE2_4X41#@M3ro`~iFp1IB0J9dHgj z3s%4*7*0M5?}ZT7z!7jbJdRP~A@~RQ5L^e>!a1-Gp2SA+1GpdF1Mh};n1gk2B>a{)up!1U}752ikZrqjqXy%hKz}!I3Bd8 zg|GoqE(Se;JXO3lvc#RuSHoOA%)0p7^W{=KoUiko#&{-XDqV$eS5}81GW}Q6!1djg zkYH!`ZOitJ44l_Lk{!KZxIepP^Z6%d`_CVCFLRlFn!H{$tj|_U!Ql4In_UpOAc+sF z)%Lc0u2d@5gWS|qInNse)8TZvx*Mq_1azzxLPxMCBN51tK1I6PW5SrJWb47|j=YMr zeSK$8D_#p^jBm&$0KRxB0k>XCuuynZ}4?pfD)nY%A0k^GEEq&lBf zBFoPZTJoebSY1z^b1`^No~Z<@A-c~cy33PzrpsGW8i-=uhRmLfk%+_~5)W3( zZ)%GEKYBlPQvKN(NCe|kxrthbz1d4(r9cb?a@tAD6zkRTUq5Lw9 z7JpCDc7#*>QJ$`3cj4kzrUb1@RK`WBR}(R5@%t?{sdLS%s{t=ePUdae zs&d9fnaC6Gg4dj?w@}Srp;$$ls^Zm1O}kgCq{&H!p{-4AHf!}SNkGSs~#n5o~({8Relg%Jxz1g-Mwk67uj;FQQUdLh zPSW*-HaB8Dg2^IV|7W~zPO{sHIprjoyUIoSDZdT#(x;xQ+cW~{#?9Uv3#o>71d7IZQRgV_W4sfKvN$fU zs9xk}4oD~FMBKFSOU6qI=W^9jd)ua5>80d=N~E;5<=JXJ45Gnw%9zl_@AoRE47sa* zW>XPN0NqHqaIUjiyfRE_Kx>NpB%LT`q)E#BQ4Y(Q`)nd;7B61_saG-0zBS z@^052dEJKVDY816Ekk|%jG=9DV4PeF>+Q*Ob_VP0a5wQJzX3H8@PfEerFSx_dG?|4 zc1yCZ>2N|WO1?5WXm2Z)cIBoh`ntR}jO|V9Kxeck6ZIN4lU!&2;B@}zNqkIuW$rxi=%$Yv~8Dy-cVR>ydy`AXO1#sxye zuTpW*3gw*W|F1#Uyajz!^#60UmE#lW_E$q524FS30G>pz|68~Urr?#Z1-jt7==Pt6 z+u?oiCb$N!h8{Q#zJqT6&u}YT2ZOK$PJ*N1F?9Q1z~|v=C_q0P4iBTxzZWK92RsLk zh9lsY==G1mFW|j!3tSA7upS;m$NxLH8}`6y@Kf~r+hIG5!r^c)y8KS~LwErF{k3of zoCZ%Iqr2f_@O}`TUdc_%taWHfpe2F*A_3RR>((8OJtOV8{Mbfy&BojqD~IT~u|_D1 zLacR;U39&le|x18mI_Vpk6lfTpi&JhxoX%fhQ#IccbEw4+0yKE6h}3jnGI|8Y*?bn z4n^~cy^;xAlVm5trY2BsP~s_swR~!H?rLfTl`^TZVXDOn8g4RGX<2g0~VySgD$ zo~?Jgb~S2Uy)ko|8P1ccHt>jj-`pzJHez<#EdP)%cC7Dm%S|`POu$S9tC@+~i%$d? zsbzya~sW`qmitSfU-TmZuxvUV^8_V-ia8k(#X- zYc`^xO&9C_>otFjvTQ{+1Tz01PqIlihA`97m!<)@U}FvGztUf` zrtA;Aj8REPhBj|Lvv#Yd0QI%3TG#zY)I_o zv4$l!O%U1S?M+JPre~#A+Dtf5wK9(m?=sg7giK*=JY%km(Yfw)x0=)DC?Zg*a8#`^ zt&bNe9pkg5Jl4!WMjX3?QqRguW>uXkc{y*Q(A8U#=vzC8usdVInB!ijlO{tRK$+7a{m!o~7*-{}yL#Fyl;Sb7{Kp8*v>_C}WPx(LPg}5y@J4GYtM=O8tb?Ten zXdW?9rWr$I&2!3xT)HI89iBv_^~%=~iIzrWyAu8X9SqqAnM)S^Kh(qbC(-ra2e-gA zFb4gw0lMKJxEG!OGjIj;!Wr4B zfY<=8f}L3dHkH_J5cr&Q{*)OWqzKqlqJ2^_x^X7$|O|0dC z(DFd=$A~Qt1iH4C2SS7MLdydIrwV1oGU}4nCz&a;vurI7god6bn)8nJH!TkY-0@l- z2zxSSfZ}H7Bv^z){rMY+4=&4Smy+Po#8Wu4VEpCXH&MjGtQ`2)^T5%LBpL z)U2b>(mW6%XI(vNuueK1=;Q_F(NO^}5pbHLy@1tjb74CuxY=zU|WH<>#7k?f+ z7e0gD{b?w{6r2cO(fapA9FK!zA%HI-uP?$Fi0rbE_T%jS)}m-;WYnw@V>zkl_F~il ziEq9*w#F{`vrx))I+_}cxgqaQFW^{W)a)4(M&e|-bIryYvw*n?oMEGX^=xH|fi1c& zzp-hX;MU(L<%VKm-UQRJnt$ge!&2TpvOGSH^HFEux_D!Mz5+>2&PZ*&(f*=gYH9U*Q-lR$wcE#!jp8 zj2Uq))foL0GToY_%d(V8Z1IJ;l01rKd9K=3m@8Kc*G@Kn~PA1zs*q80wF*MTG5u6^hd8gWfZvFq@ z&`5S$-;OrtO`1%jlyzkUTv-bwU8))d_U&SI&D6w8W&b4=XE&c6*&O@*Z;6s%1{ z!xv=F+&r}D+<-{BS#7|5-L&`1FJ;S3^gb+OMtoMVwum((Dz&IMDWsLIy>68gxiyn) z*Opy?Yz$E&rAvW5YhQl}8NY3GXrw>8Raz+Bw0QltjcBPYOoN!stD9posrxYV8()8! zx$lW$e`bF}x3=u;w=u-B-=4L(wpZCdzyE5yP2oPwbsMLBYn$s!>{g;|Z`!Bqb*;CZ z_F4UdoBE>#BR$cmcveQm${j%0hjAH$cd6ge3X_x}d}@oO;!Z2AwYg$FKWRrNzF&c< zi#CDfN;I;U+5B!jyI+brlEehRxbd`aptlm}=_UCk)*^QXC!A37pP>iVB{L_ZUH_Ym z+(iS@dVm`Lm+?7d;b zC$$p$Lv;Kfz&qeF*aAnvqv-nIf*as+sKeRN4Q&v>OW=#>|M$TZTnyXcAovz`fWLy9 zU^n!_OX1mYC>#Rf3-E2Y7d`_YfNS7t*bVb=4jd0(!)|abycKFt1=$Ng`~hT*zEv8wBtY_#$?P`(O@c z;UdVwq3|g7hdbbQkbMDeguevw6L48FICvNvg!l>E2;Fce z`6PKI>;L~xeoP%q9%r!1OwFlX=MuxUzHw|yU zwArF&8HHz)cPYl=(FzfQd?f|ppGHKtc> z$d`uu^O-d@@S!hx$c z@#O_4oDi3hZO(0nGsjL6z5y=MzGPT|^{r4GA7>|k_QKqnb~E@`-LcNgx8&_s(Ne9D z?9dQxDiKWQDizklqBy8YgkWG>P@fEgT%o{RZLMi?M2RIT))BjWm1innW9Q*+tbm@n z=2c2?R z3?s*5lRR-DFJVbtA8ZV^^q!v`+`c8-KR7xv(7#R8vWy9rqGc)R#LJM@cU-b&S+Fr7 zj^Luu#lzb2W*%X~Cep#3G7e$4CL00S6|lia!`WA@yU;6ui*NMD%tT8)8y)H!>JHdA zqc9ax5z44Xb7mqkW5Rip^X-Gt;l$9!(cVyD-VLaeU&@Z&-AR zbR%TZ|Njt0azy5@(f`lVO68^K`lrB;(dA`-zxTsMa18tyJ^mheKZLLWj)7;wH_+dA z!A8iyThYP1BM6doRTnTOPV|3`-;8r*TWbQu$_n}LF0p0~6w1e!m_X)TX&VU!f z{mA8QPy>W!04=4Nx;7pXkEpB;!ygvn#`RC1f1No#tuiELk_l8L+KsWcCws-Hc) zJ=nIT#u~d4dk4^pET3S%1mvhTD27<^0APF^jP-_^_WvmVq~Wu(y3DF zH)7@)=4cb92V~kbZ4K-Tzp05OdDm0Q)URr%D-_F`L0+J`G%tYrW7^sEV#`>_SDUTK z*I*gJxjD}G2-0MrIYv8LF&Rvu)tPFhS2@|d$ZJ~@)5ncBa!luEOnwwn{jCzhndJ0X+D3;42|Y)%d#JB%HYYQ!EZ^tiOvO@5;qU528Agt zig${}XMu-H(;|(LYJ*&{urs#HYFg}F`>aZnKGhT9J}vo6FFv-|V0p0JoSG~W$J4Q5iM>k8 zrG=Zvr$VWp#XO}Km#Ymf)GI8UY$@@2T1>XYpy)3&wgFiq@J@IQTnNvD=fZ>72|f=WfI7SqhG7U+!V}mFWWE3Wa4TE_yWw=$2&-Tv z$XfqjVN3W5d>QVByFly=MVN$jAZreu3$pG&_6qzSd>UlGK-mM}5_lEJ-2MuXegAKQ zD`6hyU>iIieueEp*8X1wyP*svcm*5*l3)M5s4XJZQju)xwe=~jMADQ|^i<8znd&*zUN8z?HjkYt1ra{flP5GH< z_niEU8WIwbmtymv-%1c_UzmjjVS+3o2$LK+X8eErzxRbuW}+`a?4>Zo5^2OM8|s+v zNi=K|)KDO1DpJ<$tLX`fSOj`6`Q~4D@UqWFWaPsO>DltQrCJi=zS&@Jvwvvgs4HR( z*3y*KJiZ99QLACEOjpPD6zVt--T~3a4eZ-UeXyzVQyFEHCqUct=}^-7Z5@t!I<=LL zjZ;fjXO(CEu~9@z)C+N+iJ3}I{Hr58+{PKpcimRd=&W*IhERrWmn*Rwbe)nL+-D;e zxI}C$miG348@KF{I;LgWXg7*-A5VZGIxRd5n5)>sR(1rk~;_VLW%Sd1J z`hLn9rREDwUa9}A-tC)5DJDr->MKr|eq6d%iCpD$clQo&eufziIf*hffjduCsirnR z?H_wDH2J*C++>kyQq@37Cp2=t`)o+sfGukO=%Pwx_d=@qa|PI!TS^mZI}^7m-@c0} zXHAOE$ocXq6_Y!sd!n|dSt;i-fU{cqosHuPs&)$5&*cuOmgj z*QrIYoRlnyzD8t|?cSZbE_rE30dD^o*ZuC(3>%dU!;Qv#I;uzez<;6l%Xk0237-U6 z3vfN`fb-x~5Z{6CV+XhyE``^?C~Six;RyHz34avCR&YOj2d;uQzy^@-0(=!4!I$A~ zxEywZ>#syJu~V-`rYI%nAL7BHwebWqgAe7v#xL2Ub=%P9?fwp} z@msY^756r}Ms=)hJQsJDipQm0xroc2pY!&}l#;HD;lA@JYKDF}H=(TWHqL!CK56Ul z)%qDfs^xlPMKbL;5Q?7apJhjG=iA*bp6hm7VCx|qJ8Rbh8OPuhZ8mK)@VF#YBc^34 zZrJjO1GWVP-GpfNz5uj~bnHzRd4I&l>IGF0M8lWZY!?|5z`|b7>5`DxJeOylwSUWm zJkKuvib=udckHZ|jn@aKU3N7%x{AM}slMh!yza&#aaGp@_LKohu&TJe{EU4-!Y@=q z)jbku@j|(O#jZ;t2fb^$X>yy4ry9IyO>f<>a>Jyz=5$(Vn#U(CXlge9H`EZuU>o|!pKg8EQ@J-2>o6S+@ zjP~VZx<&HHhFUP+Y+4($tC9_6@HobIz4*h+EN!MF=VQBwn$)PH$jCNwe$Dfi(zqSp z8g5m0$ja8$`thBC{h)!3)@*Dff#da4%rk6M6=F-60xBBssl4pzz>x$3Lbr;pVI)lYfFMzW_#A#?>$ z?3RsBinoj6BWg`2DYhJ0(G~x0wmdqq_^R_!2kWq*F`nQda3|-)T zI#>SeA33kTFC($bkGxz;a$@~JJg4NY#xrtOewL6+63ph5$qrekU|x?oP;KEg8AOi! EFH+&182|tP literal 0 HcmV?d00001 diff --git a/src/transports/mlx5-exp/.mlx5-exp.hpp.swp b/src/transports/mlx5-exp/.mlx5-exp.hpp.swp new file mode 100644 index 0000000000000000000000000000000000000000..64c77c8d59fee822a476d37dfa63f8ef615c2d7b GIT binary patch literal 16384 zcmeHO&5s<#6|X>WfCS9fDN>YTQOM3}_RG6iB*rTdc@tS;?Uike!-ATg?%LV5d%AnN zYj(XMCJKcAAQHDok&;6WK_SY4C{ZLZCk}{PawLl6gapBrj|;!oAJbJeJ+ot*1Bz;- z-}ZEUy!Yz&UR8Cyp6x81Ke@plU07oH+{4&EKm5LX=;{;qeD7VxlAfDhk~@PA)lK+> zaW{yTCp0LlUKk2o+)U$(!jsuz7<9URwjX)({_bLydW$l3qb!M2)b+y4kIak9$?(g< zPLdp~#UFsSO)$- z8IZ{>>}MF-E#>g46Tw%?`|5gYdH=2QdfuonZ|%i0U>UFsSOzQumI2FvWxz6E8L$jk z1}p=XfiEEguFu#vVAoISppWQ3YE5M7u&w=NFW58kH)4Le^1b7GdJ@88)09Ju-0-wQ{mw|V_j`qOaz`yQf>`mZz zz>B~JumU^){OS(I{(U=Rv%q(OYhPpRRX_qM&pNZ?NpV@Wc_!0=Cwb7>MOiO#yd5`+M2JSkmo}^VYAy5LXjF*_ zBFVkJXfyo8!l7SNIQTV%14WY0=|!nB_+4K@DQ%acCT52uGOjUV{KYy@& zi4d1=%(^`{kdv*`T1;o%EWD8!beLjVGInwnh-kHPR_U#llLK4bJ@}+$I$#QQW3|+VtSrpH&`EoxI{P=PHqZ4PH6K77IUq9oV`|;+wv+=}* zN1XKwnaDH5J2oV}_>YHMLUpnp$3XEt}G7W>0l!YMl}yYH>ct zb6rz-&Fl!Tm1de=qvD(PnyGH=(8z0MhTKnc^Z9Y@11@X4-HN81^BR{VomVncEYk)`gkj5>lC;1wRp!jc@VCj&r4%uB`=Q&yg0$ftHE(R)&Ql%V3%t@i^t! zOWlymm8wo$xdS%jp>L8@8{!xT{ z;`+;5%SWHBYs!Zb?8KOf*wIqHmu}Gl|J36r&p9Vg{qVH2@%RPj^yb;oPV~$}+hE?A z_Hs^tYT1g`rL9$^KaJK;gI(yPJq3p4@pO#QoiDblhpMrxOdICI{z%6ebQ6p|_ zn4eugb>e!$ym+MM#idd67ZB>|`0{;7pXIbk)vBvmmn9C%(6&>{zFs@>P0O^y>rb}B>ysT11(6pHe8Fc1G6*yJt@(^n7k{6B!s|JD2bU*jy_2NGZ%_y}kB*ML6&d%zlS9QY7t_BVi+ zfnNYG0c*hDafW{ZxB@%~yp41GGr%V}v%dn|3;Y>p_fxGhnHrS74z%pPNunbrR zECZH-FOC7qhHHnvG!Da#>s`XrtDeXx6E>839!VmpkI|Ix$?x&t79-2RMpRJZJ3rU^ zBJ;9ZOI}93NJO4t4BB}9KQL2VX5pc0L{$=Jimo(BWSrIt&CD|CErqp_yO(@Bbh}xT zTs%Jtx95x%Pmi*Ej{X{nD?Oi~s3RvJEfC@v6lOe}xT)LA4zR-Q;9o)uD$D%TFfUss zs>oX-$+IywG>y|u5_&t4_b^qA5!0?;cg*>F%H6ITHmOKbfx(Jq9mO!NXB0YR`CB9{ zzWo7S9jT^I84i@zf@1>zG6pd!7H62!6J-ERDR%g3sK^FF_Lv~fr&OFENFv8e2D*52 z{?Q~-h~P+lHBwKrqY|7;92>XI8jPTYdE?2+C6HkUn@%h+XWz@fyX6gMgl%bZN({mCn8lYgp?=FHP3H<4A1m*_EF zNcWB_Ww{>ZUzONE_o{c9x}o$!f_JZpZ@hCg-=vZ@Fp`HpDWK&O_w*ICYOX4YLx*~W zDqgQ*o2l(rETFZyPOQ*EStVT*i)oF{-v#gD|K{Uzt(}elGS3ae*rS|&zRtt^{Mu%p E1BZcreate_qp = gds_mlx5_exp_create_qp; + transport->destroy_qp = gds_mlx5_exp_destroy_qp; + transport->rollback_qp = gds_mlx5_exp_rollback_qp; + + transport->init_send_info = gds_mlx5_exp_init_send_info; + transport->post_send_ops = gds_mlx5_exp_post_send_ops; + transport->post_send_ops_on_cpu = gds_mlx5_exp_post_send_ops_on_cpu; + transport->prepare_send = gds_mlx5_exp_prepare_send; + transport->get_num_send_request_entries = gds_mlx5_exp_get_num_send_request_entries; + + transport->init_wait_request = gds_mlx5_exp_init_wait_request; + transport->dump_wait_request = gds_mlx5_exp_dump_wait_request; + transport->stream_post_wait_descriptor = gds_mlx5_exp_stream_post_wait_descriptor; + transport->post_wait_descriptor = gds_mlx5_exp_post_wait_descriptor; + transport->get_wait_descs = gds_mlx5_exp_get_wait_descs; + transport->get_num_wait_request_entries = gds_mlx5_exp_get_num_send_request_entries; + + transport->prepare_wait_cq = gds_mlx5_exp_prepare_wait_cq; + transport->append_wait_cq = gds_mlx5_exp_append_wait_cq; + transport->abort_wait_cq = gds_mlx5_exp_abort_wait_cq; + + return 0; +} + diff --git a/src/mlx5-exp.hpp b/src/transports/mlx5-exp/mlx5-exp.hpp similarity index 100% rename from src/mlx5-exp.hpp rename to src/transports/mlx5-exp/mlx5-exp.hpp From de7d52daa2025667c9e0652e606a1ad5a9418b27 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 15 Oct 2021 03:57:32 -0400 Subject: [PATCH 35/39] Fixed bugs and modified the code to use the gds_transport_t interface --- .gitignore | 3 + Makefile.am | 4 +- src/apis.cpp | 121 ++- src/gdsync.cpp | 471 +----------- src/mlx5.cpp | 120 +-- src/transport.hpp | 30 +- src/transports/mlx5-exp/.mlx5-exp.cpp.swp | Bin 40960 -> 0 bytes src/transports/mlx5-exp/.mlx5-exp.hpp.swp | Bin 16384 -> 0 bytes src/transports/mlx5-exp/mlx5-exp.cpp | 886 ++++++++++++++++++---- src/transports/mlx5-exp/mlx5-exp.hpp | 40 - src/utils.hpp | 19 +- 11 files changed, 884 insertions(+), 810 deletions(-) delete mode 100644 src/transports/mlx5-exp/.mlx5-exp.cpp.swp delete mode 100644 src/transports/mlx5-exp/.mlx5-exp.hpp.swp diff --git a/.gitignore b/.gitignore index 9a8cb0a..beb6022 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ libtool # Debug files *.dSYM/ *.su + +# Editor files +*.swp diff --git a/Makefile.am b/Makefile.am index 84a8085..61e9a2d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,8 +29,8 @@ src_libgdsync_la_LDFLAGS = -version-info @VERSION_INFO@ noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h if COMPILE_EXP_VERBS -src_libgdsync_la_SOURCES += src/mlx5-exp.cpp -noinst_HEADERS += src/mlx5-exp.hpp +src_libgdsync_la_SOURCES += src/transports/mlx5-exp/mlx5-exp.cpp +noinst_HEADERS += src/transports/mlx5-exp/mlx5-exp.hpp endif # if enabled at configure time diff --git a/src/apis.cpp b/src/apis.cpp index 69e491c..ed0a1d5 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -50,63 +50,36 @@ #include "utils.hpp" #include "archutils.h" #include "mlnxutils.h" -#include "mlx5-exp.hpp" - - -//----------------------------------------------------------------------------- - -void gds_init_ops(struct peer_op_wr *op, int count) -{ - int i = count; - while (--i) - op[i-1].next = &op[i]; - op[count-1].next = NULL; -} +#include "transport.hpp" //----------------------------------------------------------------------------- static void gds_init_send_info(gds_send_request_t *info) { - gds_mlx5_exp_send_request_t *gmexp_info; gds_dbg("send_request=%p\n", info); memset(info, 0, sizeof(*info)); - info->dtype = GDS_DRIVER_TYPE_MLX5_EXP; - - gmexp_info = to_gds_mexp_send_request(info); - - gds_mlx5_exp_init_send_info(gmexp_info); + gds_main_transport->init_send_info(info); } //----------------------------------------------------------------------------- static void gds_init_wait_request(gds_wait_request_t *request, uint32_t offset) { - gds_mlx5_exp_wait_request_t *gmexp_request; gds_dbg("wait_request=%p offset=%08x\n", request, offset); memset(request, 0, sizeof(*request)); - request->dtype = GDS_DRIVER_TYPE_MLX5_EXP; - - gmexp_request = to_gds_mexp_wait_request(request); - - gds_mlx5_exp_init_wait_request(gmexp_request, offset); + gds_main_transport->init_wait_request(request, offset); } //----------------------------------------------------------------------------- static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t *send_info) { - gds_mlx5_exp_qp_t *gmexpqp; - gds_mlx5_exp_send_request_t *gmexp_sreq; - assert(qp); assert(send_info); - gmexpqp = to_gds_mexp_qp(qp); - gmexp_sreq = to_gds_mexp_send_request(send_info); - - return gds_mlx5_exp_rollback_qp(gmexpqp, gmexp_sreq); + return gds_main_transport->rollback_qp(qp, send_info); } //----------------------------------------------------------------------------- @@ -162,20 +135,14 @@ int gds_prepare_send(struct gds_qp *gqp, gds_send_wr *p_ewr, gds_send_request_t *request) { int ret = 0; - gds_mlx5_exp_qp_t *gmexpqp; - gds_mlx5_exp_send_request_t *sreq; gds_init_send_info(request); assert(gqp); assert(gqp->qp); - assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); - - gmexpqp = to_gds_mexp_qp(gqp); - sreq = to_gds_mexp_send_request(request); - ret = gds_mlx5_exp_prepare_send(gmexpqp, p_ewr, bad_ewr, sreq); + ret = gds_main_transport->prepare_send(gqp, p_ewr, bad_ewr, request); if (ret) - gds_err("Error %d in gds_mlx5_exp_prepare_send.\n", ret); + gds_err("Error %d in prepare_send.\n", ret); return ret; } @@ -260,9 +227,6 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flags) { - gds_mlx5_exp_cq_t *gmexpcq; - gds_mlx5_exp_wait_request_t *gmexp_request; - if (flags != 0) { gds_err("invalid flags != 0\n"); return EINVAL; @@ -270,19 +234,23 @@ int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flag gds_init_wait_request(request, cq->curr_offset++); - gmexpcq = to_gds_mexp_cq(cq); - gmexp_request = to_gds_mexp_wait_request(request); - - return gds_mlx5_exp_prepare_wait_cq(gmexpcq, gmexp_request, flags); + return gds_main_transport->prepare_wait_cq(cq, request, flags); } //----------------------------------------------------------------------------- int gds_append_wait_cq(gds_wait_request_t *request, uint32_t *dw, uint32_t val) { - gds_mlx5_exp_wait_request_t *gmexp_request = to_gds_mexp_wait_request(request); + int ret = gds_transport_init(); + if (!ret) { + gds_err("error in gds_transport_init\n"); + goto out; + } - return gds_mlx5_exp_append_wait_cq(gmexp_request, dw, val); + ret = gds_main_transport->append_wait_cq(request, dw, val); + +out: + return ret; } //----------------------------------------------------------------------------- @@ -303,16 +271,10 @@ int gds_stream_post_wait_cq_all(CUstream stream, int count, gds_wait_request_t * static int gds_abort_wait_cq(struct gds_cq *cq, gds_wait_request_t *request) { - gds_mlx5_exp_cq_t *gmexpcq; - gds_mlx5_exp_wait_request_t *gmexp_request; - assert(cq); assert(request); - gmexpcq = to_gds_mexp_cq(cq); - gmexp_request = to_gds_mexp_wait_request(request); - - return gds_mlx5_exp_abort_wait_cq(gmexpcq, gmexp_request); + return gds_main_transport->abort_wait_cq(cq, request); } //----------------------------------------------------------------------------- @@ -503,14 +465,21 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem int ret = 0; n_mem_ops = 0; size_t i; - for(i = 0; i < n_descs; ++i) { + + ret = gds_transport_init(); + if (!ret) { + gds_err("error in gds_transport_init\n"); + goto out; + } + + for (i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; - switch(desc->tag) { + switch (desc->tag) { case GDS_TAG_SEND: - n_mem_ops += gds_mlx5_exp_get_num_send_request_entries(to_gds_mexp_send_request(desc->send)) + 2; // extra space, ugly + n_mem_ops += gds_main_transport->get_num_send_request_entries(desc->send) + 2; // extra space, ugly break; case GDS_TAG_WAIT: - n_mem_ops += gds_mlx5_exp_get_num_wait_request_entries(to_gds_mexp_wait_request(desc->wait)) + 2; // ditto + n_mem_ops += gds_main_transport->get_num_wait_request_entries(desc->wait) + 2; // ditto break; case GDS_TAG_WAIT_VALUE32: case GDS_TAG_WRITE_VALUE32: @@ -522,6 +491,8 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem ret = EINVAL; } } + +out: return ret; } @@ -538,6 +509,11 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ gds_peer *peer = NULL; gds_op_list_t params; + ret = gds_transport_init(); + if (!ret) { + gds_err("error in gds_transport_init\n"); + goto out; + } ret = calc_n_mem_ops(n_descs, descs, n_mem_ops); if (ret) { @@ -565,12 +541,11 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ return EINVAL; } - for(i = 0; i < n_descs; ++i) { + for (i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; - switch(desc->tag) { + switch (desc->tag) { case GDS_TAG_SEND: { - gds_mlx5_exp_send_request_t *sreq = to_gds_mexp_send_request(desc->send); - retcode = gds_mlx5_exp_post_send_ops(peer, sreq, params); + retcode = gds_main_transport->post_send_ops(peer, desc->send, params); if (retcode) { gds_err("error %d in gds_post_ops\n", retcode); ret = retcode; @@ -579,15 +554,14 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ break; } case GDS_TAG_WAIT: { - gds_mlx5_exp_wait_request_t *wreq = to_gds_mexp_wait_request(desc->wait); int flags = 0; if (move_flush && i != last_wait) { gds_dbg("discarding FLUSH!\n"); flags = GDS_POST_OPS_DISCARD_WAIT_FLUSH; } - retcode = gds_mlx5_exp_stream_post_wait_descriptor(peer, wreq, params, flags); + retcode = gds_main_transport->stream_post_wait_descriptor(peer, desc->wait, params, flags); if (retcode) { - gds_err("error %d in gds_mlx5_exp_stream_post_wait_descriptor\n", retcode); + gds_err("error %d in stream_post_wait_descriptor\n", retcode); ret = retcode; goto out; } @@ -642,13 +616,19 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) size_t i; int ret = 0; int retcode = 0; + + ret = gds_transport_init(); + if (!ret) { + gds_err("error in gds_transport_init\n"); + goto out; + } + for(i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { case GDS_TAG_SEND: { gds_dbg("desc[%zu] SEND\n", i); - gds_mlx5_exp_send_request_t *sreq = to_gds_mexp_send_request(desc->send); - retcode = gds_mlx5_exp_post_send_ops_on_cpu(sreq, flags); + retcode = gds_main_transport->post_send_ops_on_cpu(desc->send, flags); if (retcode) { gds_err("error %d in gds_post_ops_on_cpu\n", retcode); ret = retcode; @@ -658,10 +638,9 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) } case GDS_TAG_WAIT: { gds_dbg("desc[%zu] WAIT\n", i); - gds_mlx5_exp_wait_request_t *wreq = to_gds_mexp_wait_request(desc->wait); - retcode = gds_mlx5_exp_post_wait_descriptor(wreq, flags); + retcode = gds_main_transport->post_wait_descriptor(desc->wait, flags); if (retcode) { - gds_err("error %d in gds_mlx5_exp_post_wait_descriptor\n", retcode); + gds_err("error %d in post_wait_descriptor\n", retcode); ret = retcode; goto out; } diff --git a/src/gdsync.cpp b/src/gdsync.cpp index b01cd36..e9fafc8 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -43,7 +43,11 @@ #include "archutils.h" #include "mlnxutils.h" #include "task_queue.hpp" -#include "mlx5-exp.hpp" +#include "transport.hpp" + +//----------------------------------------------------------------------------- + +gds_transport_t *gds_main_transport = NULL; //----------------------------------------------------------------------------- @@ -93,10 +97,6 @@ int gds_flusher_enabled() #define CU_STREAM_BATCH_MEM_OP_RELAXED_ORDERING 0x1 #endif -// TODO: use correct value -// TODO: make it dependent upon the particular GPU -const size_t GDS_GPU_MAX_INLINE_SIZE = 256; - //----------------------------------------------------------------------------- // Note: these are default overrides, i.e. allow to disable/enable the features @@ -173,7 +173,7 @@ static bool gds_enable_inlcpy() } // simulate 64-bits writes with inlcpy -static bool gds_simulate_write64() +bool gds_simulate_write64() { static int gds_simulate_write64 = -1; if (-1 == gds_simulate_write64) { @@ -349,7 +349,7 @@ int gds_fill_membar(gds_peer *peer, gds_op_list_t &ops, int flags) //----------------------------------------------------------------------------- -static int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags) +int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags) { int retcode = 0; #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_MEMORY @@ -410,7 +410,7 @@ int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, void *ptr, const void *d //----------------------------------------------------------------------------- -static void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) +void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) { #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_MEMORY assert(param->operation == CU_STREAM_MEM_OP_WRITE_MEMORY); @@ -420,7 +420,7 @@ static void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) //----------------------------------------------------------------------------- -static int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags) +int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags) { int retcode = 0; CUdeviceptr dev_ptr = addr; @@ -467,7 +467,7 @@ int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, uint32_t *ptr, uint32_t va //----------------------------------------------------------------------------- -static int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags) +int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags) { int retcode = 0; #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_VALUE_64 @@ -581,7 +581,7 @@ unsigned poll_checker::m_global_index = 0; //----------------------------------------------------------------------------- -static int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags) +int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags) { int retcode = 0; const char *cond_str = NULL; @@ -718,270 +718,6 @@ int gds_stream_batch_ops(gds_peer *peer, CUstream stream, gds_op_list_t &ops, in //----------------------------------------------------------------------------- -/* - A) plain+membar: - WR32 - MEMBAR - WR32 - WR32 - - B) plain: - WR32 - WR32+PREBARRIER - WR32 - - C) sim64+membar: - WR32 - MEMBAR - INLCPY 8B - - D) sim64: - INLCPY 4B + POSTBARRIER - INLCPY 8B - - E) inlcpy+membar: - WR32 - MEMBAR - INLCPY XB - - F) inlcpy: - INLCPY 4B + POSTBARRIER - INLCPY 128B -*/ - -int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_list_t &ops, int post_flags) -{ - int retcode = 0; - size_t n = 0; - bool prev_was_fence = false; - bool use_inlcpy_for_dword = false; - //size_t n_ops = ops.size(); - CUstreamBatchMemOpParams param; - - gds_dbg("n_ops=%zu\n", n_ops); - - if (!peer->has_memops) { - gds_err("CUDA MemOps are required\n"); - return EINVAL; - } - - // divert the request to the same engine handling 64bits - // to avoid out-of-order execution - // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) - if (peer->has_inlcpy) { - if (!peer->has_membar) - use_inlcpy_for_dword = true; // F - } - if (gds_simulate_write64()) { - if (!peer->has_membar) { - gds_warn_once("enabling use_inlcpy_for_dword\n"); - use_inlcpy_for_dword = true; // D - } - } - - for (; op && n < n_ops; op = op->next, ++n) { - //int flags = 0; - gds_dbg("op[%zu] type:%08x\n", n, op->type); - switch(op->type) { - case GDS_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - - if (fence_op == GDS_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (!peer->has_membar) { - if (use_inlcpy_for_dword) { - assert(ops.size() > 0); - gds_dbg("patching previous param\n"); - gds_enable_barrier_for_inlcpy(&ops.back()); - } - else { - gds_dbg("recording fence event\n"); - prev_was_fence = true; - } - //retcode = 0; - } - else { - if (fence_from != GDS_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; - break; - } - int flags = 0; - if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - flags = GDS_MEMBAR_DEFAULT | GDS_MEMBAR_MLX5; - } - else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - flags = GDS_MEMBAR_SYS | GDS_MEMBAR_MLX5; - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - retcode = gds_fill_membar(peer, ops, flags); - } - } - break; - } - case GDS_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - int flags = 0; - gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%" PRIx32 "\n", dev_ptr, data); - if (use_inlcpy_for_dword) { // F || D - // membar may be out of order WRT inlcpy - if (peer->has_membar) { - gds_err("invalid feature combination, inlcpy + membar\n"); - retcode = EINVAL; - break; - } - // tail flush is set when following fence is met - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); - } - else { // A || B || C || E - // can't guarantee ordering of write32+inlcpy unless - // a membar is there - // TODO: fix driver when !weak - if (peer->has_inlcpy && !peer->has_membar) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - if (prev_was_fence) { - gds_dbg("using PRE_BARRIER as fence\n"); - flags |= GDS_WRITE_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(peer, ops, dev_ptr, data, flags); - } - break; - } - case GDS_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - int flags = 0; - gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%" PRIx64 "\n", dev_ptr, data); - // C || D - if (gds_simulate_write64()) { - // simulate 64-bit poke by inline copy - if (!peer->has_membar) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - - // tail flush is never useful here - //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); - } - else if (peer->has_write64) { - retcode = gds_fill_poke64(peer, ops, dev_ptr, data, flags); - } - else { - uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); - uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); - - if (prev_was_fence) { - gds_dbg("enabling PRE_BARRIER\n"); - flags |= GDS_WRITE_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(peer, ops, dev_ptr, datalo, flags); - - // get rid of the barrier, if there - flags &= ~GDS_WRITE_PRE_BARRIER; - - // advance to next DWORD - dev_ptr += sizeof(uint32_t); - retcode = gds_fill_poke(peer, ops, dev_ptr, datahi, flags); - } - - break; - } - case GDS_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - int flags = 0; - gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); - // catching any other size here - if (!peer->has_inlcpy) { - gds_err("inline copy is not supported\n"); - retcode = EINVAL; - break; - } - // IB Verbs bug - assert(len <= GDS_GPU_MAX_INLINE_SIZE); - //if (desc->need_flush) { - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - //} - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, src, len, flags); - break; - } - case GDS_PEER_OP_POLL_AND_DWORD: - case GDS_PEER_OP_POLL_GEQ_DWORD: - case GDS_PEER_OP_POLL_NOR_DWORD: { - int poll_cond; - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - // TODO: properly handle a following fence instead of blidly flushing - int flags = 0; - if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) - flags |= GDS_WAIT_POST_FLUSH_REMOTE; - - gds_dbg("OP_WAIT_DWORD dev_ptr=%llx data=%" PRIx32 " type=%" PRIx32 "\n", dev_ptr, data, (uint32_t)op->type); - - switch(op->type) { - case GDS_PEER_OP_POLL_NOR_DWORD: - poll_cond = GDS_WAIT_COND_NOR; - break; - case GDS_PEER_OP_POLL_GEQ_DWORD: - poll_cond = GDS_WAIT_COND_GEQ; - break; - case GDS_PEER_OP_POLL_AND_DWORD: - poll_cond = GDS_WAIT_COND_AND; - break; - default: - assert(!"cannot happen"); - retcode = EINVAL; - goto out; - } - retcode = gds_fill_poll(peer, ops, dev_ptr, data, poll_cond, flags); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - if (retcode) { - gds_err("error in fill func at entry n=%zu\n", n); - goto out; - } - } - - assert(n_ops == n); - -out: - return retcode; -} - -//----------------------------------------------------------------------------- - int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_t *dw, uint32_t val) { int retcode = 0; @@ -997,9 +733,8 @@ int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_ } for (int j=0; jpost_send_ops(peer, &info[j], ops); if (retcode) { goto out; } @@ -1026,124 +761,6 @@ int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_ //----------------------------------------------------------------------------- -int gds_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) -{ - int retcode = 0; - size_t n = 0; - gds_dbg("n_ops=%zu op=%p post_flags=0x%x\n", n_ops, op, post_flags); - for (; op && n < n_ops; op = op->next, ++n) { - //int flags = 0; - gds_dbg("op[%zu]=%p\n", n, op); - //gds_dbg("op[%zu]=%p type:%08x\n", n, op, op->type); - switch(op->type) { - case GDS_PEER_OP_FENCE: { - gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - - if (fence_op == GDS_PEER_FENCE_OP_READ) { - gds_warnc(1, "nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (fence_from != GDS_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); - retcode = EINVAL; - break; - } - if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - wmb(); - } - else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - wmb(); - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - } - break; - } - case GDS_PEER_OP_STORE_DWORD: { - uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); - uint32_t data = op->wr.dword_va.data; - // A || B || C || E - gds_dbg("STORE_DWORD ptr=%p data=%08" PRIx32 "\n", ptr, data); - gds_atomic_set(ptr, data); - break; - } - case GDS_PEER_OP_STORE_QWORD: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); - uint64_t data = op->wr.qword_va.data; - gds_dbg("STORE_QWORD ptr=%p data=%016" PRIx64 "\n", ptr, data); - gds_atomic_set(ptr, data); - break; - } - case GDS_PEER_OP_COPY_BLOCK: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); - uint64_t *src = (uint64_t*)op->wr.copy_op.src; - size_t n_bytes = op->wr.copy_op.len; - gds_dbg("COPY_BLOCK ptr=%p src=%p len=%zu\n", ptr, src, n_bytes); - gds_bf_copy(ptr, src, n_bytes); - break; - } - case GDS_PEER_OP_POLL_AND_DWORD: - case GDS_PEER_OP_POLL_GEQ_DWORD: - case GDS_PEER_OP_POLL_NOR_DWORD: { - int poll_cond; - uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); - uint32_t value = op->wr.dword_va.data; - bool flush = true; - if (post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH) - flush = false; - gds_dbg("WAIT_32 dev_ptr=%p data=%" PRIx32 " type=%" PRIx32 "\n", ptr, value, (uint32_t)op->type); - bool done = false; - do { - uint32_t data = gds_atomic_get(ptr); - switch(op->type) { - case GDS_PEER_OP_POLL_NOR_DWORD: - done = (0 != ~(data | value)); - break; - case GDS_PEER_OP_POLL_GEQ_DWORD: - done = ((int32_t)data - (int32_t)value >= 0); - break; - case GDS_PEER_OP_POLL_AND_DWORD: - done = (0 != (data & value)); - break; - default: - gds_err("invalid op type %02x\n", op->type); - retcode = EINVAL; - goto out; - } - if (done) - break; - // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric - arch_cpu_relax(); - } while(true); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - if (retcode) { - gds_err("error %d at entry n=%zu\n", retcode, n); - goto out; - } - } - -out: - return retcode; -} - -//----------------------------------------------------------------------------- - int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uint32_t val) { int retcode = 0; @@ -1152,9 +769,8 @@ int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uin assert(info); for (int j=0; jpost_send_ops_on_cpu(&info[j], 0); if (retcode) { goto out; } @@ -1174,12 +790,10 @@ int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uin void gds_dump_wait_request(gds_wait_request_t *request, size_t count) { for (size_t j = 0; j < count; ++j) { - gds_mlx5_exp_wait_request_t *gmexp_request; if (count == 0) return; - gmexp_request = to_gds_mexp_wait_request(&request[j]); - gds_mlx5_exp_dump_wait_request(gmexp_request, j); + gds_main_transport->dump_wait_request(&request[j], j); } } @@ -1628,7 +1242,7 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) { int ret = 0; - gds_mlx5_exp_qp_t *gmexpqp = NULL; + gds_qp_t *gqp = NULL; gds_peer *peer = NULL; gds_peer_attr *peer_attr = NULL; gds_driver_type dtype; @@ -1645,6 +1259,12 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, return NULL; } + ret = gds_transport_init(); + if (ret) { + gds_err("error in gds_transport_init\n"); + goto err; + } + // peer registration gds_dbg("before gds_register_peer_ex\n"); ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); @@ -1653,64 +1273,33 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, goto err; } - dtype = gds_get_driver_type(context->device); - if (dtype != GDS_DRIVER_TYPE_MLX5_EXP) { - gds_err("Unsupported IB device\n"); - goto err; - } - - gmexpqp = gds_mlx5_exp_create_qp(pd, context, qp_attr, peer, peer_attr, flags); - if (!gmexpqp) { - gds_err("Error in gds_mlx5_exp_create_qp.\n"); + ret = gds_main_transport->create_qp(pd, context, qp_attr, peer, peer_attr, flags, &gqp); + if (ret) { + gds_err("Error in create_qp.\n"); goto err; } - gds_dbg("created gds_qp=%p\n", gmexpqp->gqp); + gds_dbg("created gds_qp=%p\n", gqp); - return &gmexpqp->gqp; + return gqp; err: return NULL; } -//----------------------------------------------------------------------------- - -int gds_destroy_cq(struct gds_cq *gcq) -{ - int retcode = 0; - int ret; - - if (!gcq) - return retcode; - - // Currently, we support only exp-verbs. - assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); - - gds_mlx5_exp_cq_t *gmexpcq = to_gds_mexp_cq(gcq); - - retcode = gds_mlx5_exp_destroy_cq(gmexpcq); - - return retcode; -} //----------------------------------------------------------------------------- int gds_destroy_qp(struct gds_qp *gqp) { - int retcode = 0; - int ret; + int ret = 0; if (!gqp) - return retcode; - - // Currently, we support only exp-verbs. - assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); + return ret; - gds_mlx5_exp_qp_t *gmexpqp = to_gds_mexp_qp(gqp); + ret = gds_main_transport->destroy_qp(gqp); - retcode = gds_mlx5_exp_destroy_qp(gmexpqp); - - return retcode; + return ret; } //----------------------------------------------------------------------------- diff --git a/src/mlx5.cpp b/src/mlx5.cpp index c67a2f6..94872cc 100644 --- a/src/mlx5.cpp +++ b/src/mlx5.cpp @@ -40,7 +40,7 @@ //#include "mem.hpp" #include "objs.hpp" #include "utils.hpp" -#include "mlx5-exp.hpp" +#include "transport.hpp" #if 0 union { uint64_t qw; uint32_t dw[2]; } db_val; @@ -50,123 +50,21 @@ mlx5_i->db_value = db_val.qw; #endif -//----------------------------------------------------------------------------- - -int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *_request) -{ - int retcode = 0; - const gds_mlx5_exp_send_request_t *request = to_gds_mexp_send_request(_request); - size_t n_ops = request->commit.entries; - peer_op_wr *op = request->commit.storage; - size_t n = 0; - - memset(mlx5_i, 0, sizeof(*mlx5_i)); - - for (; op && n < n_ops; op = op->next, ++n) { - switch(op->type) { - case GDS_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); - if (fence_op == GDS_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); - break; - } - if (fence_from != GDS_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; - break; - } - if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - mlx5_i->membar = 1; - } - else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - mlx5_i->membar_full = 1; - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - break; - } - case GDS_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - if (n != 0) { - gds_err("store DWORD is not 1st op\n"); - retcode = EINVAL; - break; - } - mlx5_i->dbrec_ptr = (uint32_t*)dev_ptr; - mlx5_i->dbrec_value = data; - break; - } - case GDS_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); - if (n != 2) { - gds_err("store QWORD is not 3rd op\n"); - retcode = EINVAL; - break; - } - mlx5_i->db_ptr = (uint64_t*)dev_ptr; - mlx5_i->db_value = data; - break; - } - case GDS_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - gds_dbg("send inline detected\n"); - if (len < 8 || len > 64) { - gds_err("unexpected len %zu\n", len); - retcode = EINVAL; - break; - } - mlx5_i->db_ptr = (uint64_t*)dev_ptr; - mlx5_i->db_value = *(uint64_t*)src; - break; - } - case GDS_PEER_OP_POLL_AND_DWORD: - case GDS_PEER_OP_POLL_GEQ_DWORD: - case GDS_PEER_OP_POLL_NOR_DWORD: { - gds_err("unexpected polling op in send request\n"); - retcode = EINVAL; - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - - if (retcode) { - gds_err("error in fill func at entry n=%zu\n", n); - break; - } - } - return retcode; -} //----------------------------------------------------------------------------- int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_mlx5_send_info_t *mlx5_infos) { - int retcode = 0; + int retcode = gds_transport_init(); + if (retcode) { + gds_err("error in gds_transport_init\n"); + goto out; + } for (int j=0; jget_send_descs(mlx5_i, request); if (retcode) { gds_err("error %d while retrieving descriptors for %dth request\n", retcode, j); break; @@ -175,6 +73,7 @@ int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_ml mlx5_i->dbrec_ptr, mlx5_i->dbrec_value, mlx5_i->db_ptr, mlx5_i->db_value); } +out: return retcode; } @@ -182,8 +81,7 @@ int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_ml int gds_mlx5_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *request) { - const gds_mlx5_exp_wait_request_t *gmexp_request = to_gds_mexp_wait_request(request); - return gds_mlx5_exp_get_wait_descs(mlx5_i, gmexp_request); + return gds_main_transport->get_wait_descs(mlx5_i, request); } //----------------------------------------------------------------------------- diff --git a/src/transport.hpp b/src/transport.hpp index ddef072..d14d9e3 100644 --- a/src/transport.hpp +++ b/src/transport.hpp @@ -27,6 +27,10 @@ #pragma once +#include +#include +#include + typedef struct gds_transport { int (*create_qp)(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, gds_peer *peer, gds_peer_attr *peer_attr, int flags, gds_qp_t **gqp); int (*destroy_qp)(gds_qp_t *gqp); @@ -34,8 +38,9 @@ typedef struct gds_transport { void (*init_send_info)(gds_send_request_t *request); int (*post_send_ops)(gds_peer *peer, gds_send_request_t *request, gds_op_list_t &ops); - int (*post_send_ops_on_cpu)(gds_send_request_t *request, int flags = 0); + int (*post_send_ops_on_cpu)(gds_send_request_t *request, int flags); int (*prepare_send)(gds_qp_t *gqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request); + int (*get_send_descs)(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *_request); uint32_t (*get_num_send_request_entries)(gds_send_request_t *request); void (*init_wait_request)(gds_wait_request_t *request, uint32_t offset); @@ -45,14 +50,31 @@ typedef struct gds_transport { int (*get_wait_descs)(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *request); uint32_t (*get_num_wait_request_entries)(gds_wait_request_t *request); - int (*prepare_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request, in flags); - int (*append_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request, in flags); + int (*prepare_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request, int flags); + int (*append_wait_cq)(gds_wait_request_t *request, uint32_t *dw, uint32_t val); int (*abort_wait_cq)(gds_cq_t *gcq, gds_wait_request_t *request); } gds_transport_t; extern gds_transport_t *gds_main_transport; -int gds_transport_mlx5_exp_init(gds_transport_t *transport); +int gds_transport_mlx5_exp_init(gds_transport_t **transport); + +static inline int gds_transport_init() +{ + int status = 0; + if (!gds_main_transport) { + gds_transport_t *t = NULL; + status = gds_transport_mlx5_exp_init(&t); + if (status) { + gds_err("error in gds_transport_mlx5_exp_init\n"); + goto out; + } + assert(t); + gds_main_transport = t; + } +out: + return status; +} /* * Local variables: diff --git a/src/transports/mlx5-exp/.mlx5-exp.cpp.swp b/src/transports/mlx5-exp/.mlx5-exp.cpp.swp deleted file mode 100644 index 09e0b7542379f4809868a318355bb10c5039cc58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmeI53v^{wd4R8#hoihKJ}cHkV8&#~WM<@LGh`r40z_t#OeVvCikF*vPm;?d_ukxl zGs!SKl-kuUDX2?%)S^;R)CDdpw4zAWiqfhT-;XL#T3^*lwUr`>O22=f{W|BKo5wga zW!amxzMSNoefHVs-_QTw|K4Y8-FX9Bf|qx#%kXnlCUfo2CUZBQ_^D_A(U&rr%5<)} zvpzXHCf64DnJP~dOY0X1pf6o+BV0RNEnghw>$SC0#j%M(ZFecZrm$;mt(sq3ujWd% zO1a9l>8bgX*M##G`$t#4Qd#^ltq?5c_Pm8yx-n+fzfePPQb_ zl0ZuWEeW(F(2_t)0xb!&B+!z;@1_Lmm1ktG;MIOlzuuwx__W0FApL)+kIzV4H~$y) z`KiS96^Zj-Ph3A(|G!xudvyX#Iu6n2C-w2_#C4PIv-I`bkNuVWxmIQt$C6Fs*GHZ^^WI~Ynf1^kLg#Yh@ z+u&As3tSEoZWSC4hrvTfWHR@_N8uxICG^22`1RqL%qQUEkcB>Y2K?qQuED>-L+~%~ zMz{v@a56mptW4%@uoH@~5ncp`z`^j7XJ#_rfRDjlFb&(`cz6~(4Knae3=e+;cfdUK z!oOpzxE?NpQ{XW80x$kfxEB5#reOvA2*bmhVIE2_4X41#@M3ro`~iFp1IB0J9dHgj z3s%4*7*0M5?}ZT7z!7jbJdRP~A@~RQ5L^e>!a1-Gp2SA+1GpdF1Mh};n1gk2B>a{)up!1U}752ikZrqjqXy%hKz}!I3Bd8 zg|GoqE(Se;JXO3lvc#RuSHoOA%)0p7^W{=KoUiko#&{-XDqV$eS5}81GW}Q6!1djg zkYH!`ZOitJ44l_Lk{!KZxIepP^Z6%d`_CVCFLRlFn!H{$tj|_U!Ql4In_UpOAc+sF z)%Lc0u2d@5gWS|qInNse)8TZvx*Mq_1azzxLPxMCBN51tK1I6PW5SrJWb47|j=YMr zeSK$8D_#p^jBm&$0KRxB0k>XCuuynZ}4?pfD)nY%A0k^GEEq&lBf zBFoPZTJoebSY1z^b1`^No~Z<@A-c~cy33PzrpsGW8i-=uhRmLfk%+_~5)W3( zZ)%GEKYBlPQvKN(NCe|kxrthbz1d4(r9cb?a@tAD6zkRTUq5Lw9 z7JpCDc7#*>QJ$`3cj4kzrUb1@RK`WBR}(R5@%t?{sdLS%s{t=ePUdae zs&d9fnaC6Gg4dj?w@}Srp;$$ls^Zm1O}kgCq{&H!p{-4AHf!}SNkGSs~#n5o~({8Relg%Jxz1g-Mwk67uj;FQQUdLh zPSW*-HaB8Dg2^IV|7W~zPO{sHIprjoyUIoSDZdT#(x;xQ+cW~{#?9Uv3#o>71d7IZQRgV_W4sfKvN$fU zs9xk}4oD~FMBKFSOU6qI=W^9jd)ua5>80d=N~E;5<=JXJ45Gnw%9zl_@AoRE47sa* zW>XPN0NqHqaIUjiyfRE_Kx>NpB%LT`q)E#BQ4Y(Q`)nd;7B61_saG-0zBS z@^052dEJKVDY816Ekk|%jG=9DV4PeF>+Q*Ob_VP0a5wQJzX3H8@PfEerFSx_dG?|4 zc1yCZ>2N|WO1?5WXm2Z)cIBoh`ntR}jO|V9Kxeck6ZIN4lU!&2;B@}zNqkIuW$rxi=%$Yv~8Dy-cVR>ydy`AXO1#sxye zuTpW*3gw*W|F1#Uyajz!^#60UmE#lW_E$q524FS30G>pz|68~Urr?#Z1-jt7==Pt6 z+u?oiCb$N!h8{Q#zJqT6&u}YT2ZOK$PJ*N1F?9Q1z~|v=C_q0P4iBTxzZWK92RsLk zh9lsY==G1mFW|j!3tSA7upS;m$NxLH8}`6y@Kf~r+hIG5!r^c)y8KS~LwErF{k3of zoCZ%Iqr2f_@O}`TUdc_%taWHfpe2F*A_3RR>((8OJtOV8{Mbfy&BojqD~IT~u|_D1 zLacR;U39&le|x18mI_Vpk6lfTpi&JhxoX%fhQ#IccbEw4+0yKE6h}3jnGI|8Y*?bn z4n^~cy^;xAlVm5trY2BsP~s_swR~!H?rLfTl`^TZVXDOn8g4RGX<2g0~VySgD$ zo~?Jgb~S2Uy)ko|8P1ccHt>jj-`pzJHez<#EdP)%cC7Dm%S|`POu$S9tC@+~i%$d? zsbzya~sW`qmitSfU-TmZuxvUV^8_V-ia8k(#X- zYc`^xO&9C_>otFjvTQ{+1Tz01PqIlihA`97m!<)@U}FvGztUf` zrtA;Aj8REPhBj|Lvv#Yd0QI%3TG#zY)I_o zv4$l!O%U1S?M+JPre~#A+Dtf5wK9(m?=sg7giK*=JY%km(Yfw)x0=)DC?Zg*a8#`^ zt&bNe9pkg5Jl4!WMjX3?QqRguW>uXkc{y*Q(A8U#=vzC8usdVInB!ijlO{tRK$+7a{m!o~7*-{}yL#Fyl;Sb7{Kp8*v>_C}WPx(LPg}5y@J4GYtM=O8tb?Ten zXdW?9rWr$I&2!3xT)HI89iBv_^~%=~iIzrWyAu8X9SqqAnM)S^Kh(qbC(-ra2e-gA zFb4gw0lMKJxEG!OGjIj;!Wr4B zfY<=8f}L3dHkH_J5cr&Q{*)OWqzKqlqJ2^_x^X7$|O|0dC z(DFd=$A~Qt1iH4C2SS7MLdydIrwV1oGU}4nCz&a;vurI7god6bn)8nJH!TkY-0@l- z2zxSSfZ}H7Bv^z){rMY+4=&4Smy+Po#8Wu4VEpCXH&MjGtQ`2)^T5%LBpL z)U2b>(mW6%XI(vNuueK1=;Q_F(NO^}5pbHLy@1tjb74CuxY=zU|WH<>#7k?f+ z7e0gD{b?w{6r2cO(fapA9FK!zA%HI-uP?$Fi0rbE_T%jS)}m-;WYnw@V>zkl_F~il ziEq9*w#F{`vrx))I+_}cxgqaQFW^{W)a)4(M&e|-bIryYvw*n?oMEGX^=xH|fi1c& zzp-hX;MU(L<%VKm-UQRJnt$ge!&2TpvOGSH^HFEux_D!Mz5+>2&PZ*&(f*=gYH9U*Q-lR$wcE#!jp8 zj2Uq))foL0GToY_%d(V8Z1IJ;l01rKd9K=3m@8Kc*G@Kn~PA1zs*q80wF*MTG5u6^hd8gWfZvFq@ z&`5S$-;OrtO`1%jlyzkUTv-bwU8))d_U&SI&D6w8W&b4=XE&c6*&O@*Z;6s%1{ z!xv=F+&r}D+<-{BS#7|5-L&`1FJ;S3^gb+OMtoMVwum((Dz&IMDWsLIy>68gxiyn) z*Opy?Yz$E&rAvW5YhQl}8NY3GXrw>8Raz+Bw0QltjcBPYOoN!stD9posrxYV8()8! zx$lW$e`bF}x3=u;w=u-B-=4L(wpZCdzyE5yP2oPwbsMLBYn$s!>{g;|Z`!Bqb*;CZ z_F4UdoBE>#BR$cmcveQm${j%0hjAH$cd6ge3X_x}d}@oO;!Z2AwYg$FKWRrNzF&c< zi#CDfN;I;U+5B!jyI+brlEehRxbd`aptlm}=_UCk)*^QXC!A37pP>iVB{L_ZUH_Ym z+(iS@dVm`Lm+?7d;b zC$$p$Lv;Kfz&qeF*aAnvqv-nIf*as+sKeRN4Q&v>OW=#>|M$TZTnyXcAovz`fWLy9 zU^n!_OX1mYC>#Rf3-E2Y7d`_YfNS7t*bVb=4jd0(!)|abycKFt1=$Ng`~hT*zEv8wBtY_#$?P`(O@c z;UdVwq3|g7hdbbQkbMDeguevw6L48FICvNvg!l>E2;Fce z`6PKI>;L~xeoP%q9%r!1OwFlX=MuxUzHw|yU zwArF&8HHz)cPYl=(FzfQd?f|ppGHKtc> z$d`uu^O-d@@S!hx$c z@#O_4oDi3hZO(0nGsjL6z5y=MzGPT|^{r4GA7>|k_QKqnb~E@`-LcNgx8&_s(Ne9D z?9dQxDiKWQDizklqBy8YgkWG>P@fEgT%o{RZLMi?M2RIT))BjWm1innW9Q*+tbm@n z=2c2?R z3?s*5lRR-DFJVbtA8ZV^^q!v`+`c8-KR7xv(7#R8vWy9rqGc)R#LJM@cU-b&S+Fr7 zj^Luu#lzb2W*%X~Cep#3G7e$4CL00S6|lia!`WA@yU;6ui*NMD%tT8)8y)H!>JHdA zqc9ax5z44Xb7mqkW5Rip^X-Gt;l$9!(cVyD-VLaeU&@Z&-AR zbR%TZ|Njt0azy5@(f`lVO68^K`lrB;(dA`-zxTsMa18tyJ^mheKZLLWj)7;wH_+dA z!A8iyThYP1BM6doRTnTOPV|3`-;8r*TWbQu$_n}LF0p0~6w1e!m_X)TX&VU!f z{mA8QPy>W!04=4Nx;7pXkEpB;!ygvn#`RC1f1No#tuiELk_l8L+KsWcCws-Hc) zJ=nIT#u~d4dk4^pET3S%1mvhTD27<^0APF^jP-_^_WvmVq~Wu(y3DF zH)7@)=4cb92V~kbZ4K-Tzp05OdDm0Q)URr%D-_F`L0+J`G%tYrW7^sEV#`>_SDUTK z*I*gJxjD}G2-0MrIYv8LF&Rvu)tPFhS2@|d$ZJ~@)5ncBa!luEOnwwn{jCzhndJ0X+D3;42|Y)%d#JB%HYYQ!EZ^tiOvO@5;qU528Agt zig${}XMu-H(;|(LYJ*&{urs#HYFg}F`>aZnKGhT9J}vo6FFv-|V0p0JoSG~W$J4Q5iM>k8 zrG=Zvr$VWp#XO}Km#Ymf)GI8UY$@@2T1>XYpy)3&wgFiq@J@IQTnNvD=fZ>72|f=WfI7SqhG7U+!V}mFWWE3Wa4TE_yWw=$2&-Tv z$XfqjVN3W5d>QVByFly=MVN$jAZreu3$pG&_6qzSd>UlGK-mM}5_lEJ-2MuXegAKQ zD`6hyU>iIieueEp*8X1wyP*svcm*5*l3)M5s4XJZQju)xwe=~jMADQ|^i<8znd&*zUN8z?HjkYt1ra{flP5GH< z_niEU8WIwbmtymv-%1c_UzmjjVS+3o2$LK+X8eErzxRbuW}+`a?4>Zo5^2OM8|s+v zNi=K|)KDO1DpJ<$tLX`fSOj`6`Q~4D@UqWFWaPsO>DltQrCJi=zS&@Jvwvvgs4HR( z*3y*KJiZ99QLACEOjpPD6zVt--T~3a4eZ-UeXyzVQyFEHCqUct=}^-7Z5@t!I<=LL zjZ;fjXO(CEu~9@z)C+N+iJ3}I{Hr58+{PKpcimRd=&W*IhERrWmn*Rwbe)nL+-D;e zxI}C$miG348@KF{I;LgWXg7*-A5VZGIxRd5n5)>sR(1rk~;_VLW%Sd1J z`hLn9rREDwUa9}A-tC)5DJDr->MKr|eq6d%iCpD$clQo&eufziIf*hffjduCsirnR z?H_wDH2J*C++>kyQq@37Cp2=t`)o+sfGukO=%Pwx_d=@qa|PI!TS^mZI}^7m-@c0} zXHAOE$ocXq6_Y!sd!n|dSt;i-fU{cqosHuPs&)$5&*cuOmgj z*QrIYoRlnyzD8t|?cSZbE_rE30dD^o*ZuC(3>%dU!;Qv#I;uzez<;6l%Xk0237-U6 z3vfN`fb-x~5Z{6CV+XhyE``^?C~Six;RyHz34avCR&YOj2d;uQzy^@-0(=!4!I$A~ zxEywZ>#syJu~V-`rYI%nAL7BHwebWqgAe7v#xL2Ub=%P9?fwp} z@msY^756r}Ms=)hJQsJDipQm0xroc2pY!&}l#;HD;lA@JYKDF}H=(TWHqL!CK56Ul z)%qDfs^xlPMKbL;5Q?7apJhjG=iA*bp6hm7VCx|qJ8Rbh8OPuhZ8mK)@VF#YBc^34 zZrJjO1GWVP-GpfNz5uj~bnHzRd4I&l>IGF0M8lWZY!?|5z`|b7>5`DxJeOylwSUWm zJkKuvib=udckHZ|jn@aKU3N7%x{AM}slMh!yza&#aaGp@_LKohu&TJe{EU4-!Y@=q z)jbku@j|(O#jZ;t2fb^$X>yy4ry9IyO>f<>a>Jyz=5$(Vn#U(CXlge9H`EZuU>o|!pKg8EQ@J-2>o6S+@ zjP~VZx<&HHhFUP+Y+4($tC9_6@HobIz4*h+EN!MF=VQBwn$)PH$jCNwe$Dfi(zqSp z8g5m0$ja8$`thBC{h)!3)@*Dff#da4%rk6M6=F-60xBBssl4pzz>x$3Lbr;pVI)lYfFMzW_#A#?>$ z?3RsBinoj6BWg`2DYhJ0(G~x0wmdqq_^R_!2kWq*F`nQda3|-)T zI#>SeA33kTFC($bkGxz;a$@~JJg4NY#xrtOewL6+63ph5$qrekU|x?oP;KEg8AOi! EFH+&182|tP diff --git a/src/transports/mlx5-exp/.mlx5-exp.hpp.swp b/src/transports/mlx5-exp/.mlx5-exp.hpp.swp deleted file mode 100644 index 64c77c8d59fee822a476d37dfa63f8ef615c2d7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHO&5s<#6|X>WfCS9fDN>YTQOM3}_RG6iB*rTdc@tS;?Uike!-ATg?%LV5d%AnN zYj(XMCJKcAAQHDok&;6WK_SY4C{ZLZCk}{PawLl6gapBrj|;!oAJbJeJ+ot*1Bz;- z-}ZEUy!Yz&UR8Cyp6x81Ke@plU07oH+{4&EKm5LX=;{;qeD7VxlAfDhk~@PA)lK+> zaW{yTCp0LlUKk2o+)U$(!jsuz7<9URwjX)({_bLydW$l3qb!M2)b+y4kIak9$?(g< zPLdp~#UFsSO)$- z8IZ{>>}MF-E#>g46Tw%?`|5gYdH=2QdfuonZ|%i0U>UFsSOzQumI2FvWxz6E8L$jk z1}p=XfiEEguFu#vVAoISppWQ3YE5M7u&w=NFW58kH)4Le^1b7GdJ@88)09Ju-0-wQ{mw|V_j`qOaz`yQf>`mZz zz>B~JumU^){OS(I{(U=Rv%q(OYhPpRRX_qM&pNZ?NpV@Wc_!0=Cwb7>MOiO#yd5`+M2JSkmo}^VYAy5LXjF*_ zBFVkJXfyo8!l7SNIQTV%14WY0=|!nB_+4K@DQ%acCT52uGOjUV{KYy@& zi4d1=%(^`{kdv*`T1;o%EWD8!beLjVGInwnh-kHPR_U#llLK4bJ@}+$I$#QQW3|+VtSrpH&`EoxI{P=PHqZ4PH6K77IUq9oV`|;+wv+=}* zN1XKwnaDH5J2oV}_>YHMLUpnp$3XEt}G7W>0l!YMl}yYH>ct zb6rz-&Fl!Tm1de=qvD(PnyGH=(8z0MhTKnc^Z9Y@11@X4-HN81^BR{VomVncEYk)`gkj5>lC;1wRp!jc@VCj&r4%uB`=Q&yg0$ftHE(R)&Ql%V3%t@i^t! zOWlymm8wo$xdS%jp>L8@8{!xT{ z;`+;5%SWHBYs!Zb?8KOf*wIqHmu}Gl|J36r&p9Vg{qVH2@%RPj^yb;oPV~$}+hE?A z_Hs^tYT1g`rL9$^KaJK;gI(yPJq3p4@pO#QoiDblhpMrxOdICI{z%6ebQ6p|_ zn4eugb>e!$ym+MM#idd67ZB>|`0{;7pXIbk)vBvmmn9C%(6&>{zFs@>P0O^y>rb}B>ysT11(6pHe8Fc1G6*yJt@(^n7k{6B!s|JD2bU*jy_2NGZ%_y}kB*ML6&d%zlS9QY7t_BVi+ zfnNYG0c*hDafW{ZxB@%~yp41GGr%V}v%dn|3;Y>p_fxGhnHrS74z%pPNunbrR zECZH-FOC7qhHHnvG!Da#>s`XrtDeXx6E>839!VmpkI|Ix$?x&t79-2RMpRJZJ3rU^ zBJ;9ZOI}93NJO4t4BB}9KQL2VX5pc0L{$=Jimo(BWSrIt&CD|CErqp_yO(@Bbh}xT zTs%Jtx95x%Pmi*Ej{X{nD?Oi~s3RvJEfC@v6lOe}xT)LA4zR-Q;9o)uD$D%TFfUss zs>oX-$+IywG>y|u5_&t4_b^qA5!0?;cg*>F%H6ITHmOKbfx(Jq9mO!NXB0YR`CB9{ zzWo7S9jT^I84i@zf@1>zG6pd!7H62!6J-ERDR%g3sK^FF_Lv~fr&OFENFv8e2D*52 z{?Q~-h~P+lHBwKrqY|7;92>XI8jPTYdE?2+C6HkUn@%h+XWz@fyX6gMgl%bZN({mCn8lYgp?=FHP3H<4A1m*_EF zNcWB_Ww{>ZUzONE_o{c9x}o$!f_JZpZ@hCg-=vZ@Fp`HpDWK&O_w*ICYOX4YLx*~W zDqgQ*o2l(rETFZyPOQ*EStVT*i)oF{-v#gD|K{Uzt(}elGS3ae*rS|&zRtt^{Mu%p E1BZhas_memops) { + gds_err("CUDA MemOps are required\n"); + return EINVAL; + } + + // divert the request to the same engine handling 64bits + // to avoid out-of-order execution + // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) + if (peer->has_inlcpy) { + if (!peer->has_membar) + use_inlcpy_for_dword = true; // F + } + if (gds_simulate_write64()) { + if (!peer->has_membar) { + gds_warn_once("enabling use_inlcpy_for_dword\n"); + use_inlcpy_for_dword = true; // D + } + } + + for (; op && n < n_ops; op = op->next, ++n) { + //int flags = 0; + gds_dbg("op[%zu] type:%08x\n", n, op->type); + switch(op->type) { + case GDS_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + //retcode = EINVAL; + break; + } + else { + if (!peer->has_membar) { + if (use_inlcpy_for_dword) { + assert(ops.size() > 0); + gds_dbg("patching previous param\n"); + gds_enable_barrier_for_inlcpy(&ops.back()); + } + else { + gds_dbg("recording fence event\n"); + prev_was_fence = true; + } + //retcode = 0; + } + else { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + int flags = 0; + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + flags = GDS_MEMBAR_DEFAULT | GDS_MEMBAR_MLX5; + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + flags = GDS_MEMBAR_SYS | GDS_MEMBAR_MLX5; + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + retcode = gds_fill_membar(peer, ops, flags); + } + } + break; + } + case GDS_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + int flags = 0; + gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%" PRIx32 "\n", dev_ptr, data); + if (use_inlcpy_for_dword) { // F || D + // membar may be out of order WRT inlcpy + if (peer->has_membar) { + gds_err("invalid feature combination, inlcpy + membar\n"); + retcode = EINVAL; + break; + } + // tail flush is set when following fence is met + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); + } + else { // A || B || C || E + // can't guarantee ordering of write32+inlcpy unless + // a membar is there + // TODO: fix driver when !weak + if (peer->has_inlcpy && !peer->has_membar) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + if (prev_was_fence) { + gds_dbg("using PRE_BARRIER as fence\n"); + flags |= GDS_WRITE_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(peer, ops, dev_ptr, data, flags); + } + break; + } + case GDS_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + int flags = 0; + gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%" PRIx64 "\n", dev_ptr, data); + // C || D + if (gds_simulate_write64()) { + // simulate 64-bit poke by inline copy + if (!peer->has_membar) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + + // tail flush is never useful here + //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); + } + else if (peer->has_write64) { + retcode = gds_fill_poke64(peer, ops, dev_ptr, data, flags); + } + else { + uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); + uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); + + if (prev_was_fence) { + gds_dbg("enabling PRE_BARRIER\n"); + flags |= GDS_WRITE_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(peer, ops, dev_ptr, datalo, flags); + + // get rid of the barrier, if there + flags &= ~GDS_WRITE_PRE_BARRIER; + + // advance to next DWORD + dev_ptr += sizeof(uint32_t); + retcode = gds_fill_poke(peer, ops, dev_ptr, datahi, flags); + } + + break; + } + case GDS_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + int flags = 0; + gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); + // catching any other size here + if (!peer->has_inlcpy) { + gds_err("inline copy is not supported\n"); + retcode = EINVAL; + break; + } + // IB Verbs bug + assert(len <= GDS_GPU_MAX_INLINE_SIZE); + //if (desc->need_flush) { + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + //} + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, src, len, flags); + break; + } + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { + int poll_cond; + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + // TODO: properly handle a following fence instead of blidly flushing + int flags = 0; + if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) + flags |= GDS_WAIT_POST_FLUSH_REMOTE; + + gds_dbg("OP_WAIT_DWORD dev_ptr=%llx data=%" PRIx32 " type=%" PRIx32 "\n", dev_ptr, data, (uint32_t)op->type); + + switch(op->type) { + case GDS_PEER_OP_POLL_NOR_DWORD: + poll_cond = GDS_WAIT_COND_NOR; + break; + case GDS_PEER_OP_POLL_GEQ_DWORD: + poll_cond = GDS_WAIT_COND_GEQ; + break; + case GDS_PEER_OP_POLL_AND_DWORD: + poll_cond = GDS_WAIT_COND_AND; + break; + default: + assert(!"cannot happen"); + retcode = EINVAL; + goto out; + } + retcode = gds_fill_poll(peer, ops, dev_ptr, data, poll_cond, flags); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + if (retcode) { + gds_err("error in fill func at entry n=%zu\n", n); + goto out; + } + } + + assert(n_ops == n); + +out: + return retcode; +} + +//----------------------------------------------------------------------------- + +static int gds_mlx5_exp_post_ops_on_cpu(size_t n_ops, struct peer_op_wr *op, int post_flags) +{ + int retcode = 0; + size_t n = 0; + gds_dbg("n_ops=%zu op=%p post_flags=0x%x\n", n_ops, op, post_flags); + for (; op && n < n_ops; op = op->next, ++n) { + //int flags = 0; + gds_dbg("op[%zu]=%p\n", n, op); + //gds_dbg("op[%zu]=%p type:%08x\n", n, op, op->type); + switch(op->type) { + case GDS_PEER_OP_FENCE: { + gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_warnc(1, "nothing to do for read fences\n"); + //retcode = EINVAL; + break; + } + else { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); + retcode = EINVAL; + break; + } + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + wmb(); + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + wmb(); + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + } + break; + } + case GDS_PEER_OP_STORE_DWORD: { + uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); + uint32_t data = op->wr.dword_va.data; + // A || B || C || E + gds_dbg("STORE_DWORD ptr=%p data=%08" PRIx32 "\n", ptr, data); + gds_atomic_set(ptr, data); + break; + } + case GDS_PEER_OP_STORE_QWORD: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); + uint64_t data = op->wr.qword_va.data; + gds_dbg("STORE_QWORD ptr=%p data=%016" PRIx64 "\n", ptr, data); + gds_atomic_set(ptr, data); + break; + } + case GDS_PEER_OP_COPY_BLOCK: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); + uint64_t *src = (uint64_t*)op->wr.copy_op.src; + size_t n_bytes = op->wr.copy_op.len; + gds_dbg("COPY_BLOCK ptr=%p src=%p len=%zu\n", ptr, src, n_bytes); + gds_bf_copy(ptr, src, n_bytes); + break; + } + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { + int poll_cond; + uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); + uint32_t value = op->wr.dword_va.data; + bool flush = true; + if (post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH) + flush = false; + gds_dbg("WAIT_32 dev_ptr=%p data=%" PRIx32 " type=%" PRIx32 "\n", ptr, value, (uint32_t)op->type); + bool done = false; + do { + uint32_t data = gds_atomic_get(ptr); + switch(op->type) { + case GDS_PEER_OP_POLL_NOR_DWORD: + done = (0 != ~(data | value)); + break; + case GDS_PEER_OP_POLL_GEQ_DWORD: + done = ((int32_t)data - (int32_t)value >= 0); + break; + case GDS_PEER_OP_POLL_AND_DWORD: + done = (0 != (data & value)); + break; + default: + gds_err("invalid op type %02x\n", op->type); + retcode = EINVAL; + goto out; + } + if (done) + break; + // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric + arch_cpu_relax(); + } while(true); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + if (retcode) { + gds_err("error %d at entry n=%zu\n", retcode, n); + goto out; + } + } + +out: + return retcode; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *_request) +{ + int retcode = 0; + const gds_mlx5_exp_send_request_t *request = to_gds_mexp_send_request(_request); + size_t n_ops = request->commit.entries; + peer_op_wr *op = request->commit.storage; + size_t n = 0; + + memset(mlx5_i, 0, sizeof(*mlx5_i)); + + for (; op && n < n_ops; op = op->next, ++n) { + switch(op->type) { + case GDS_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + break; + } + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + mlx5_i->membar = 1; + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + mlx5_i->membar_full = 1; + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + break; + } + case GDS_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + if (n != 0) { + gds_err("store DWORD is not 1st op\n"); + retcode = EINVAL; + break; + } + mlx5_i->dbrec_ptr = (uint32_t*)dev_ptr; + mlx5_i->dbrec_value = data; + break; + } + case GDS_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); + if (n != 2) { + gds_err("store QWORD is not 3rd op\n"); + retcode = EINVAL; + break; + } + mlx5_i->db_ptr = (uint64_t*)dev_ptr; + mlx5_i->db_value = data; + break; + } + case GDS_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + gds_dbg("send inline detected\n"); + if (len < 8 || len > 64) { + gds_err("unexpected len %zu\n", len); + retcode = EINVAL; + break; + } + mlx5_i->db_ptr = (uint64_t*)dev_ptr; + mlx5_i->db_value = *(uint64_t*)src; + break; + } + case GDS_PEER_OP_POLL_AND_DWORD: + case GDS_PEER_OP_POLL_GEQ_DWORD: + case GDS_PEER_OP_POLL_NOR_DWORD: { + gds_err("unexpected polling op in send request\n"); + retcode = EINVAL; + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + + if (retcode) { + gds_err("error in fill func at entry n=%zu\n", n); + break; + } + } + return retcode; +} + +//----------------------------------------------------------------------------- + static ibv_exp_res_domain *gds_mlx5_exp_create_res_domain(struct ibv_context *context) { if (!context) { @@ -77,9 +580,89 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( //----------------------------------------------------------------------------- -gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( +int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq) +{ + int retcode = 0; + int ret; + + if (!gmexpcq) + return retcode; + + assert(gmexpcq->gcq.dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + if (gmexpcq->gcq.cq) { + ret = ibv_destroy_cq(gmexpcq->gcq.cq); + if (ret) { + gds_err("error %d in destroy_cq\n", ret); + retcode = ret; + } + } + + // res_domain will be destroyed in gds_mlx5_exp_destroy_qp. + + free(gmexpcq); + + return retcode; +} + + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_destroy_qp(gds_qp_t *gqp) +{ + int retcode = 0; + int ret; + + if (!gqp) + return retcode; + + gds_mlx5_exp_qp_t *gmexpqp = to_gds_mexp_qp(gqp); + + assert(gmexpqp->gqp.dtype == GDS_DRIVER_TYPE_MLX5_EXP); + + if (gmexpqp->gqp.qp) { + ret = ibv_destroy_qp(gmexpqp->gqp.qp); + if (ret) { + gds_err("error %d in destroy_qp\n", ret); + retcode = ret; + } + } + + if (gmexpqp->gqp.send_cq) { + ret = gds_mlx5_exp_destroy_cq(to_gds_mexp_cq(gmexpqp->gqp.send_cq)); + if (ret) { + gds_err("error %d in destroy_cq send_cq\n", ret); + retcode = ret; + } + } + + if (gmexpqp->gqp.recv_cq) { + ret = gds_mlx5_exp_destroy_cq(to_gds_mexp_cq(gmexpqp->gqp.recv_cq)); + if (ret) { + gds_err("error %d in destroy_cq recv_cq\n", ret); + retcode = ret; + } + } + + if (gmexpqp->res_domain) { + struct ibv_exp_destroy_res_domain_attr attr = {0,}; //IBV_EXP_DESTROY_RES_DOMAIN_RESERVED + ret = ibv_exp_destroy_res_domain(gmexpqp->gqp.dev_context, gmexpqp->res_domain, &attr); + if (ret) { + gds_err("ibv_exp_destroy_res_domain error %d: %s\n", ret, strerror(ret)); + retcode = ret; + } + } + + free(gmexpqp); + + return retcode; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_exp_create_qp( struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, - gds_peer *peer, gds_peer_attr *peer_attr, int flags) + gds_peer *peer, gds_peer_attr *peer_attr, int flags, gds_qp_t **gqp) { int ret = 0; gds_mlx5_exp_qp_t *gmexpqp = NULL; @@ -96,8 +679,9 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( gmexpqp = (gds_mlx5_exp_qp_t *)calloc(1, sizeof(gds_mlx5_exp_qp_t)); if (!gmexpqp) { - gds_err("cannot allocate memory\n"); - return NULL; + ret = ENOMEM; + gds_err("cannot allocate memory\n"); + goto err; } gmexpqp->gqp.dtype = GDS_DRIVER_TYPE_MLX5_EXP; @@ -171,99 +755,37 @@ gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( gds_dbg("created gds_mlx5_exp_qp=%p\n", gmexpqp); - return gmexpqp; - -err: - gds_dbg("destroying QP\n"); - gds_mlx5_exp_destroy_qp(gmexpqp); - - return NULL; -} - -//----------------------------------------------------------------------------- - -int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp) -{ - int retcode = 0; - int ret; - - if (!gmexpqp) - return retcode; - - assert(gmexpqp->gqp.dtype == GDS_DRIVER_TYPE_MLX5_EXP); - - if (gmexpqp->gqp.qp) { - ret = ibv_destroy_qp(gmexpqp->gqp.qp); - if (ret) { - gds_err("error %d in destroy_qp\n", ret); - retcode = ret; - } - } - - if (gmexpqp->gqp.send_cq) { - ret = gds_destroy_cq(gmexpqp->gqp.send_cq); - if (ret) { - gds_err("error %d in destroy_cq send_cq\n", ret); - retcode = ret; - } - } + *gqp = &gmexpqp->gqp; - if (gmexpqp->gqp.recv_cq) { - ret = gds_destroy_cq(gmexpqp->gqp.recv_cq); - if (ret) { - gds_err("error %d in destroy_cq recv_cq\n", ret); - retcode = ret; - } - } + return 0; - if (gmexpqp->res_domain) { - struct ibv_exp_destroy_res_domain_attr attr = {0,}; //IBV_EXP_DESTROY_RES_DOMAIN_RESERVED - ret = ibv_exp_destroy_res_domain(gmexpqp->gqp.dev_context, gmexpqp->res_domain, &attr); - if (ret) { - gds_err("ibv_exp_destroy_res_domain error %d: %s\n", ret, strerror(ret)); - retcode = ret; - } +err: + if (gmexpqp) { + gds_dbg("destroying QP\n"); + gds_mlx5_exp_destroy_qp(&gmexpqp->gqp); } - free(gmexpqp); - - return retcode; + return ret; } + //----------------------------------------------------------------------------- -int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq) +int gds_mlx5_exp_prepare_send(gds_qp_t *gqp, gds_send_wr *p_ewr, + gds_send_wr **bad_ewr, + gds_send_request_t *_request) { - int retcode = 0; - int ret; - - if (!gmexpcq) - return retcode; - - assert(gmexpcq->gcq.dtype == GDS_DRIVER_TYPE_MLX5_EXP); - - if (gmexpcq->gcq.cq) { - ret = ibv_destroy_cq(gmexpcq->gcq.cq); - if (ret) { - gds_err("error %d in destroy_cq\n", ret); - retcode = ret; - } - } - - // res_domain will be destroyed in gds_mlx5_exp_destroy_qp. + int ret = 0; - free(gmexpcq); + gds_mlx5_exp_qp_t *gmexpqp; + gds_mlx5_exp_send_request_t *request; - return retcode; -} + assert(gqp); + assert(_request); -//----------------------------------------------------------------------------- + gmexpqp = to_gds_mexp_qp(gqp); + request = to_gds_mexp_send_request(_request); -int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, - gds_send_wr **bad_ewr, - gds_mlx5_exp_send_request_t *request) -{ - int ret = 0; ret = ibv_post_send(gmexpqp->gqp.qp, p_ewr, bad_ewr); if (ret) { @@ -287,39 +809,60 @@ int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, //----------------------------------------------------------------------------- -void gds_mlx5_exp_init_send_info(gds_mlx5_exp_send_request_t *info) +void gds_mlx5_exp_init_send_info(gds_send_request_t *_info) { + gds_mlx5_exp_send_request_t *info; + + assert(_info); + info = to_gds_mexp_send_request(_info); + gds_dbg("send_request=%p\n", info); info->commit.storage = info->wr; info->commit.entries = sizeof(info->wr)/sizeof(info->wr[0]); - gds_init_ops(info->commit.storage, info->commit.entries); + gds_mlx5_exp_init_ops(info->commit.storage, info->commit.entries); } //----------------------------------------------------------------------------- -int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_mlx5_exp_send_request_t *info, gds_op_list_t &ops) +int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_send_request_t *_info, gds_op_list_t &ops) { - return gds_post_ops(peer, info->commit.entries, info->commit.storage, ops, 0); + gds_mlx5_exp_send_request_t *info; + + assert(peer); + assert(_info); + + info = to_gds_mexp_send_request(_info); + return gds_mlx5_exp_post_ops(peer, info->commit.entries, info->commit.storage, ops, 0); } //----------------------------------------------------------------------------- -int gds_mlx5_exp_post_send_ops_on_cpu(gds_mlx5_exp_send_request_t *info, int flags) +int gds_mlx5_exp_post_send_ops_on_cpu(gds_send_request_t *_info, int flags) { - return gds_post_ops_on_cpu(info->commit.entries, info->commit.storage, flags); + gds_mlx5_exp_send_request_t *info; + + assert(_info); + + info = to_gds_mexp_send_request(_info); + return gds_mlx5_exp_post_ops_on_cpu(info->commit.entries, info->commit.storage, flags); } //----------------------------------------------------------------------------- -void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset) +void gds_mlx5_exp_init_wait_request(gds_wait_request_t *_request, uint32_t offset) { + gds_mlx5_exp_wait_request_t *request; + + assert(_request); + request = to_gds_mexp_wait_request(_request); + gds_dbg("wait_request=%p offset=%08x\n", request, offset); request->peek.storage = request->wr; request->peek.entries = sizeof(request->wr)/sizeof(request->wr[0]); request->peek.whence = IBV_EXP_PEER_PEEK_ABSOLUTE; request->peek.offset = offset; - gds_init_ops(request->peek.storage, request->peek.entries); + gds_mlx5_exp_init_ops(request->peek.storage, request->peek.entries); } //----------------------------------------------------------------------------- @@ -382,9 +925,14 @@ static void gds_mlx5_exp_dump_ops(struct peer_op_wr *op, size_t count) //----------------------------------------------------------------------------- -void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t idx) +void gds_mlx5_exp_dump_wait_request(gds_wait_request_t *_request, size_t idx) { - struct ibv_exp_peer_peek *peek = &request->peek; + gds_mlx5_exp_wait_request_t *request; + struct ibv_exp_peer_peek *peek; + + assert(_request); + request = to_gds_mexp_wait_request(_request); + peek = &request->peek; gds_dbg("req[%zu] entries:%u whence:%u offset:%u peek_id:%" PRIx64 " comp_mask:%08x\n", idx, peek->entries, peek->whence, peek->offset, peek->peek_id, peek->comp_mask); @@ -393,9 +941,17 @@ void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t //----------------------------------------------------------------------------- -int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_request_t *request, int flags) +int gds_mlx5_exp_prepare_wait_cq(gds_cq_t *gcq, gds_wait_request_t *_request, int flags) { int retcode = 0; + gds_mlx5_exp_cq_t *mexpcq; + gds_mlx5_exp_wait_request_t *request; + + assert(gcq); + assert(_request); + + mexpcq = to_gds_mexp_cq(gcq); + request = to_gds_mexp_wait_request(_request); retcode = ibv_exp_peer_peek_cq(mexpcq->gcq.cq, &request->peek); if (retcode == ENOSPC) { @@ -413,12 +969,20 @@ int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_re //----------------------------------------------------------------------------- -int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t *dw, uint32_t val) +int gds_mlx5_exp_append_wait_cq(gds_wait_request_t *_request, uint32_t *dw, uint32_t val) { int ret = 0; - unsigned MAX_NUM_ENTRIES = sizeof(request->wr) / sizeof(request->wr[0]); - unsigned n = request->peek.entries; - struct peer_op_wr *wr = request->peek.storage; + unsigned MAX_NUM_ENTRIES; + unsigned n; + struct peer_op_wr *wr; + gds_mlx5_exp_wait_request_t *request; + + assert(_request); + + request = to_gds_mexp_wait_request(_request); + MAX_NUM_ENTRIES = sizeof(request->wr) / sizeof(request->wr[0]); + n = request->peek.entries; + wr = request->peek.storage; if (n + 1 > MAX_NUM_ENTRIES) { gds_err("no space left to stuff a poke\n"); @@ -448,9 +1012,18 @@ int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t * //----------------------------------------------------------------------------- -int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_request_t *request) +int gds_mlx5_exp_abort_wait_cq(gds_cq_t *gcq, gds_wait_request_t *_request) { struct ibv_exp_peer_abort_peek abort_ctx; + gds_mlx5_exp_cq_t *gmexpcq; + gds_mlx5_exp_wait_request_t *request; + + assert(gcq); + assert(_request); + + gmexpcq = to_gds_mexp_cq(gcq); + request = to_gds_mexp_wait_request(_request); + abort_ctx.peek_id = request->peek.peek_id; abort_ctx.comp_mask = 0; return ibv_exp_peer_abort_peek_cq(gmexpcq->gcq.cq, &abort_ctx); @@ -458,35 +1031,46 @@ int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_req //----------------------------------------------------------------------------- -int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_mlx5_exp_wait_request_t *request, gds_op_list_t ¶ms, int flags) +int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_wait_request_t *_request, gds_op_list_t ¶ms, int flags) { int ret = 0; + gds_mlx5_exp_wait_request_t *request; - ret = gds_post_ops(peer, request->peek.entries, request->peek.storage, params, flags); + assert(peer); + assert(_request); + + request = to_gds_mexp_wait_request(_request); + + ret = gds_mlx5_exp_post_ops(peer, request->peek.entries, request->peek.storage, params, flags); if (ret) - gds_err("error %d in gds_post_ops\n", ret); + gds_err("error %d in gds_mlx5_exp_post_ops\n", ret); return ret; } //----------------------------------------------------------------------------- -int gds_mlx5_exp_post_wait_descriptor(gds_mlx5_exp_wait_request_t *request, int flags) +int gds_mlx5_exp_post_wait_descriptor(gds_wait_request_t *_request, int flags) { int ret = 0; + gds_mlx5_exp_wait_request_t *request; - ret = gds_post_ops_on_cpu(request->peek.entries, request->peek.storage, flags); + assert(_request); + request = to_gds_mexp_wait_request(_request); + + ret = gds_mlx5_exp_post_ops_on_cpu(request->peek.entries, request->peek.storage, flags); if (ret) - gds_err("error %d in gds_post_ops_on_cpu\n", ret); + gds_err("error %d in gds_mlx5_exp_post_ops_on_cpu\n", ret); return ret; } //----------------------------------------------------------------------------- -int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp_wait_request_t *request) +int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *_request) { int retcode = 0; + const gds_mlx5_exp_wait_request_t *request = to_gds_mexp_wait_request(_request); size_t n_ops = request->peek.entries; peer_op_wr *op = request->peek.storage; size_t n = 0; @@ -591,15 +1175,21 @@ int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp //----------------------------------------------------------------------------- -int gds_mlx5_exp_rollback_qp(gds_mlx5_exp_qp_t *gmexpqp, gds_mlx5_exp_send_request_t *send_info) +int gds_mlx5_exp_rollback_qp(gds_qp_t *gqp, gds_send_request_t *request) { struct ibv_exp_rollback_ctx rollback; int ret = 0; enum ibv_exp_rollback_flags flag = IBV_EXP_ROLLBACK_ABORT_LATE; + gds_mlx5_exp_send_request_t *send_info; + + gds_mlx5_exp_qp_t *gmexpqp; + + assert(gqp); + assert(gqp->qp); + assert(request); - assert(gmexpqp); - assert(gmexpqp->gqp.qp); - assert(send_info); + gmexpqp = to_gds_mexp_qp(gqp); + send_info = to_gds_mexp_send_request(request); /* from ibv_exp_peer_commit call */ rollback.rollback_id = send_info->commit.rollback_id; @@ -618,29 +1208,59 @@ int gds_mlx5_exp_rollback_qp(gds_mlx5_exp_qp_t *gmexpqp, gds_mlx5_exp_send_reque //----------------------------------------------------------------------------- -int gds_transport_mlx5_exp_init(gds_transport_t *transport) +uint32_t gds_mlx5_exp_get_num_wait_request_entries(gds_wait_request_t *request) { + gds_mlx5_exp_wait_request_t *gmexp_request; + assert(request); + gmexp_request = to_gds_mexp_wait_request(request); + return gmexp_request->peek.entries; +} + +//----------------------------------------------------------------------------- + +uint32_t gds_mlx5_exp_get_num_send_request_entries(gds_send_request_t *request) { + gds_mlx5_exp_send_request_t *gmexp_request; + assert(request); + gmexp_request = to_gds_mexp_send_request(request); + return gmexp_request->commit.entries; +} + +//----------------------------------------------------------------------------- + +int gds_transport_mlx5_exp_init(gds_transport_t **transport) { - transport->create_qp = gds_mlx5_exp_create_qp; - transport->destroy_qp = gds_mlx5_exp_destroy_qp; - transport->rollback_qp = gds_mlx5_exp_rollback_qp; - - transport->init_send_info = gds_mlx5_exp_init_send_info; - transport->post_send_ops = gds_mlx5_exp_post_send_ops; - transport->post_send_ops_on_cpu = gds_mlx5_exp_post_send_ops_on_cpu; - transport->prepare_send = gds_mlx5_exp_prepare_send; - transport->get_num_send_request_entries = gds_mlx5_exp_get_num_send_request_entries; - - transport->init_wait_request = gds_mlx5_exp_init_wait_request; - transport->dump_wait_request = gds_mlx5_exp_dump_wait_request; - transport->stream_post_wait_descriptor = gds_mlx5_exp_stream_post_wait_descriptor; - transport->post_wait_descriptor = gds_mlx5_exp_post_wait_descriptor; - transport->get_wait_descs = gds_mlx5_exp_get_wait_descs; - transport->get_num_wait_request_entries = gds_mlx5_exp_get_num_send_request_entries; - - transport->prepare_wait_cq = gds_mlx5_exp_prepare_wait_cq; - transport->append_wait_cq = gds_mlx5_exp_append_wait_cq; - transport->abort_wait_cq = gds_mlx5_exp_abort_wait_cq; + int status = 0; - return 0; + gds_transport_t *t = (gds_transport_t *)calloc(1, sizeof(gds_transport_t)); + if (!t) { + status = ENOMEM; + goto out; + } + + t->create_qp = gds_mlx5_exp_create_qp; + t->destroy_qp = gds_mlx5_exp_destroy_qp; + t->rollback_qp = gds_mlx5_exp_rollback_qp; + + t->init_send_info = gds_mlx5_exp_init_send_info; + t->post_send_ops = gds_mlx5_exp_post_send_ops; + t->post_send_ops_on_cpu = gds_mlx5_exp_post_send_ops_on_cpu; + t->prepare_send = gds_mlx5_exp_prepare_send; + t->get_send_descs = gds_mlx5_exp_get_send_descs; + t->get_num_send_request_entries = gds_mlx5_exp_get_num_send_request_entries; + + t->init_wait_request = gds_mlx5_exp_init_wait_request; + t->dump_wait_request = gds_mlx5_exp_dump_wait_request; + t->stream_post_wait_descriptor = gds_mlx5_exp_stream_post_wait_descriptor; + t->post_wait_descriptor = gds_mlx5_exp_post_wait_descriptor; + t->get_wait_descs = gds_mlx5_exp_get_wait_descs; + t->get_num_wait_request_entries = gds_mlx5_exp_get_num_wait_request_entries; + + t->prepare_wait_cq = gds_mlx5_exp_prepare_wait_cq; + t->append_wait_cq = gds_mlx5_exp_append_wait_cq; + t->abort_wait_cq = gds_mlx5_exp_abort_wait_cq; + + *transport = t; + +out: + return status; } diff --git a/src/transports/mlx5-exp/mlx5-exp.hpp b/src/transports/mlx5-exp/mlx5-exp.hpp index 1d67137..e4f792d 100644 --- a/src/transports/mlx5-exp/mlx5-exp.hpp +++ b/src/transports/mlx5-exp/mlx5-exp.hpp @@ -72,43 +72,3 @@ static inline const gds_mlx5_exp_wait_request_t *to_gds_mexp_wait_request(const return (const gds_mlx5_exp_wait_request_t *)to_gds_mexp_wait_request((const gds_wait_request_t *)gwreq); } -static inline uint32_t gds_mlx5_exp_get_num_wait_request_entries(gds_mlx5_exp_wait_request_t *gmexp_request) { - return gmexp_request->peek.entries; -} - -static inline uint32_t gds_mlx5_exp_get_num_send_request_entries(gds_mlx5_exp_send_request_t *gmexp_request) { - return gmexp_request->commit.entries; -} - -gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( - struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, gds_peer *peer, gds_peer_attr *peer_attr, gds_alloc_cq_flags_t flags, - struct ibv_exp_res_domain *res_domain); - -gds_mlx5_exp_qp_t *gds_mlx5_exp_create_qp( - struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_attr, - gds_peer *peer, gds_peer_attr *peer_attr, int flags); - -int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq); -int gds_mlx5_exp_destroy_qp(gds_mlx5_exp_qp_t *gmexpqp); - -int gds_mlx5_exp_prepare_send(gds_mlx5_exp_qp_t *gmexpqp, gds_send_wr *p_ewr, - gds_send_wr **bad_ewr, - gds_mlx5_exp_send_request_t *request); - - -void gds_mlx5_exp_init_send_info(gds_mlx5_exp_send_request_t *info); -int gds_mlx5_exp_post_send_ops(gds_peer *peer, gds_mlx5_exp_send_request_t *info, gds_op_list_t &ops); -int gds_mlx5_exp_post_send_ops_on_cpu(gds_mlx5_exp_send_request_t *info, int flags = 0); - -void gds_mlx5_exp_init_wait_request(gds_mlx5_exp_wait_request_t *request, uint32_t offset); -void gds_mlx5_exp_dump_wait_request(gds_mlx5_exp_wait_request_t *request, size_t idx); -int gds_mlx5_exp_prepare_wait_cq(gds_mlx5_exp_cq_t *mexpcq, gds_mlx5_exp_wait_request_t *request, int flags); -int gds_mlx5_exp_append_wait_cq(gds_mlx5_exp_wait_request_t *request, uint32_t *dw, uint32_t val); -int gds_mlx5_exp_abort_wait_cq(gds_mlx5_exp_cq_t *gmexpcq, gds_mlx5_exp_wait_request_t *request); -int gds_mlx5_exp_stream_post_wait_descriptor(gds_peer *peer, gds_mlx5_exp_wait_request_t *request, gds_op_list_t ¶ms, int flags); -int gds_mlx5_exp_post_wait_descriptor(gds_mlx5_exp_wait_request_t *request, int flags); -int gds_mlx5_exp_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_exp_wait_request_t *request); - -int gds_mlx5_exp_rollback_qp(gds_mlx5_exp_qp_t *gmexpqp, gds_mlx5_exp_send_request_t *send_info); diff --git a/src/utils.hpp b/src/utils.hpp index c5d0774..a146f60 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -191,6 +191,10 @@ typedef enum gds_alloc_qp_flags { #include +// TODO: use correct value +// TODO: make it dependent upon the particular GPU +const size_t GDS_GPU_MAX_INLINE_SIZE = 256; + typedef std::vector gds_op_list_t; struct gds_cq *gds_create_cq(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags); @@ -205,9 +209,13 @@ struct gds_peer; int gds_fill_membar(gds_peer *peer, gds_op_list_t ¶m, int flags); int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t ¶m, void *ptr, const void *data, size_t n_bytes, int flags); +int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags); int gds_fill_poke(gds_peer *peer, gds_op_list_t ¶m, uint32_t *ptr, uint32_t value, int flags); +int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags); int gds_fill_poke64(gds_peer *peer, gds_op_list_t ¶m, uint64_t *ptr, uint64_t value, int flags); +int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags); int gds_fill_poll(gds_peer *peer, gds_op_list_t ¶m, uint32_t *ptr, uint32_t magic, int cond_flag, int flags); +int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags); int gds_stream_batch_ops(gds_peer *peer, CUstream stream, gds_op_list_t ¶ms, int flags); @@ -216,10 +224,11 @@ enum gds_post_ops_flags { }; struct gds_peer; -int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_list_t ¶ms, int post_flags = 0); -int gds_post_ops_on_cpu(size_t n_descs, struct peer_op_wr *op, int post_flags = 0); gds_peer *peer_from_stream(CUstream stream); +bool gds_simulate_write64(); +void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param); + //----------------------------------------------------------------------------- /* \brief: Get the underlying driver associated with the ibdev. @@ -238,12 +247,6 @@ static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) //----------------------------------------------------------------------------- -int gds_destroy_cq(struct gds_cq *gcq); - -void gds_init_ops(struct peer_op_wr *op, int count); - -//----------------------------------------------------------------------------- - /* * Local variables: * c-indent-level: 8 From c914e89f99543c87f2ebd4d50400ed1f2738ee48 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 15 Oct 2021 04:37:53 -0400 Subject: [PATCH 36/39] Removed driver_type and fixed bugs --- include/gdsync/core.h | 17 ++--------------- src/apis.cpp | 8 ++++---- src/gdsync.cpp | 1 - src/transports/mlx5-exp/mlx5-exp.cpp | 7 ------- src/transports/mlx5-exp/mlx5-exp.hpp | 14 ++------------ src/utils.hpp | 16 ---------------- 6 files changed, 8 insertions(+), 55 deletions(-) diff --git a/include/gdsync/core.h b/include/gdsync/core.h index fa213d3..e500c93 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -57,17 +57,9 @@ enum gds_create_qp_flags { typedef struct ibv_qp_init_attr gds_qp_init_attr_t; typedef struct ibv_send_wr gds_send_wr; -typedef enum gds_driver_type { - GDS_DRIVER_TYPE_UNSUPPORTED = 0, - GDS_DRIVER_TYPE_MLX5_EXP, - GDS_DRIVER_TYPE_MLX5_DV, - GDS_DRIVER_TYPE_MLX5_DEVX -} gds_driver_type_t; - typedef struct gds_cq { struct ibv_cq *cq; uint32_t curr_offset; - gds_driver_type_t dtype; } gds_cq_t; typedef struct gds_qp { @@ -75,7 +67,6 @@ typedef struct gds_qp { struct gds_cq *send_cq; struct gds_cq *recv_cq; struct ibv_context *dev_context; - gds_driver_type_t dtype; } gds_qp_t; /* \brief: Create a peer-enabled QP attached to the specified GPU id. @@ -161,11 +152,9 @@ enum { */ typedef struct gds_send_request { - gds_driver_type_t dtype; - uint8_t pad0[4]; uint8_t reserved0[32]; uint8_t reserved1[56 * GDS_SEND_INFO_MAX_OPS]; - uint8_t pad1[24]; + uint8_t pad0[32]; } gds_send_request_t; static_assert(sizeof(gds_send_request_t) % 64 == 0, "gds_send_request_t must be 64-byte aligned."); @@ -179,11 +168,9 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req */ typedef struct gds_wait_request { - gds_driver_type_t dtype; - uint8_t pad0[4]; uint8_t reserved0[40]; uint8_t reserved1[56 * GDS_WAIT_INFO_MAX_OPS]; - uint8_t pad1[16]; + uint8_t pad0[24]; } gds_wait_request_t; static_assert(sizeof(gds_wait_request_t) % 64 == 0, "gds_wait_request_t must be 64-byte aligned."); diff --git a/src/apis.cpp b/src/apis.cpp index ed0a1d5..cf9008c 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -242,7 +242,7 @@ int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flag int gds_append_wait_cq(gds_wait_request_t *request, uint32_t *dw, uint32_t val) { int ret = gds_transport_init(); - if (!ret) { + if (ret) { gds_err("error in gds_transport_init\n"); goto out; } @@ -467,7 +467,7 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem size_t i; ret = gds_transport_init(); - if (!ret) { + if (ret) { gds_err("error in gds_transport_init\n"); goto out; } @@ -510,7 +510,7 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ gds_op_list_t params; ret = gds_transport_init(); - if (!ret) { + if (ret) { gds_err("error in gds_transport_init\n"); goto out; } @@ -618,7 +618,7 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) int retcode = 0; ret = gds_transport_init(); - if (!ret) { + if (ret) { gds_err("error in gds_transport_init\n"); goto out; } diff --git a/src/gdsync.cpp b/src/gdsync.cpp index e9fafc8..84a9b05 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -1245,7 +1245,6 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_t *gqp = NULL; gds_peer *peer = NULL; gds_peer_attr *peer_attr = NULL; - gds_driver_type dtype; int old_errno = errno; gds_dbg("pd=%p context=%p gpu_id=%d flags=%08x current errno=%d\n", pd, context, gpu_id, flags, errno); diff --git a/src/transports/mlx5-exp/mlx5-exp.cpp b/src/transports/mlx5-exp/mlx5-exp.cpp index 23abb7a..85f97e2 100644 --- a/src/transports/mlx5-exp/mlx5-exp.cpp +++ b/src/transports/mlx5-exp/mlx5-exp.cpp @@ -573,8 +573,6 @@ gds_mlx5_exp_cq_t *gds_mlx5_exp_create_cq( return NULL; } - gmexpcq->gcq.dtype = GDS_DRIVER_TYPE_MLX5_EXP; - return gmexpcq; } @@ -588,8 +586,6 @@ int gds_mlx5_exp_destroy_cq(gds_mlx5_exp_cq_t *gmexpcq) if (!gmexpcq) return retcode; - assert(gmexpcq->gcq.dtype == GDS_DRIVER_TYPE_MLX5_EXP); - if (gmexpcq->gcq.cq) { ret = ibv_destroy_cq(gmexpcq->gcq.cq); if (ret) { @@ -618,8 +614,6 @@ int gds_mlx5_exp_destroy_qp(gds_qp_t *gqp) gds_mlx5_exp_qp_t *gmexpqp = to_gds_mexp_qp(gqp); - assert(gmexpqp->gqp.dtype == GDS_DRIVER_TYPE_MLX5_EXP); - if (gmexpqp->gqp.qp) { ret = ibv_destroy_qp(gmexpqp->gqp.qp); if (ret) { @@ -683,7 +677,6 @@ int gds_mlx5_exp_create_qp( gds_err("cannot allocate memory\n"); goto err; } - gmexpqp->gqp.dtype = GDS_DRIVER_TYPE_MLX5_EXP; gmexpqp->gqp.dev_context = context; diff --git a/src/transports/mlx5-exp/mlx5-exp.hpp b/src/transports/mlx5-exp/mlx5-exp.hpp index e4f792d..861c12e 100644 --- a/src/transports/mlx5-exp/mlx5-exp.hpp +++ b/src/transports/mlx5-exp/mlx5-exp.hpp @@ -23,39 +23,30 @@ typedef struct gds_mlx5_exp_qp { } gds_mlx5_exp_qp_t; typedef struct gds_mlx5_exp_send_request { - gds_driver_type_t dtype; - uint8_t pad0[4]; struct ibv_exp_peer_commit commit; struct peer_op_wr wr[GDS_SEND_INFO_MAX_OPS]; - uint8_t pad1[24]; + uint8_t pad1[32]; } gds_mlx5_exp_send_request_t; static_assert(sizeof(gds_mlx5_exp_send_request_t) % 64 == 0, "gds_mlx5_exp_send_request_t must be 64-byte aligned."); static_assert(sizeof(gds_mlx5_exp_send_request_t) <= sizeof(gds_send_request_t), "The size of gds_mlx5_exp_send_request_t must be less than or equal to that of gds_send_request_t."); -static_assert(offsetof(gds_mlx5_exp_send_request_t, dtype) == offsetof(gds_send_request_t, dtype), "dtype of gds_mlx5_exp_send_request_t and gds_send_request_t must be at the same offset."); typedef struct gds_mlx5_exp_wait_request { - gds_driver_type_t dtype; - uint8_t pad0[4]; struct ibv_exp_peer_peek peek; struct peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; - uint8_t pad1[16]; + uint8_t pad1[24]; } gds_mlx5_exp_wait_request_t; static_assert(sizeof(gds_mlx5_exp_wait_request_t) % 64 == 0, "gds_mlx5_exp_wait_request_t must be 64-byte aligned."); static_assert(sizeof(gds_mlx5_exp_wait_request_t) <= sizeof(gds_wait_request_t), "The size of gds_mlx5_exp_wait_request_t must be less than or equal to that of gds_wait_request_t."); -static_assert(offsetof(gds_mlx5_exp_wait_request_t, dtype) == offsetof(gds_wait_request_t, dtype), "dtype of gds_mlx5_exp_wait_request_t and gds_wait_request_t must be at the same offset."); static inline gds_mlx5_exp_cq_t *to_gds_mexp_cq(gds_cq_t *gcq) { - assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return container_of(gcq, gds_mlx5_exp_cq_t, gcq); } static inline gds_mlx5_exp_qp_t *to_gds_mexp_qp(gds_qp_t *gqp) { - assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return container_of(gqp, gds_mlx5_exp_qp_t, gqp); } static inline gds_mlx5_exp_send_request_t *to_gds_mexp_send_request(gds_send_request_t *gsreq) { - assert(gsreq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return (gds_mlx5_exp_send_request_t *)(gsreq); } @@ -64,7 +55,6 @@ static inline const gds_mlx5_exp_send_request_t *to_gds_mexp_send_request(const } static inline gds_mlx5_exp_wait_request_t *to_gds_mexp_wait_request(gds_wait_request_t *gwreq) { - assert(gwreq->dtype == GDS_DRIVER_TYPE_MLX5_EXP); return (gds_mlx5_exp_wait_request_t *)(gwreq); } diff --git a/src/utils.hpp b/src/utils.hpp index a146f60..cd2102e 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -231,22 +231,6 @@ void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param); //----------------------------------------------------------------------------- -/* \brief: Get the underlying driver associated with the ibdev. - * - */ -static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) -{ - const char *dev_name = ibv_get_device_name(ibdev); - - // Heuristically guess the driver by the device name. - // Until we find a better way to do so... - if (strstr(dev_name, "mlx5") != NULL) - return GDS_DRIVER_TYPE_MLX5_EXP; - return GDS_DRIVER_TYPE_UNSUPPORTED; -} - -//----------------------------------------------------------------------------- - /* * Local variables: * c-indent-level: 8 From ed703ccf3482c895998601d28c07387d00c34351 Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Fri, 15 Oct 2021 04:42:31 -0400 Subject: [PATCH 37/39] Added exp-verbs support checking in transport.hpp --- src/transport.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/transport.hpp b/src/transport.hpp index d14d9e3..00078cb 100644 --- a/src/transport.hpp +++ b/src/transport.hpp @@ -27,6 +27,7 @@ #pragma once +#include #include #include #include @@ -57,19 +58,28 @@ typedef struct gds_transport { extern gds_transport_t *gds_main_transport; +#if HAVE_EXP_VERBS int gds_transport_mlx5_exp_init(gds_transport_t **transport); +#else +#warning "This library requires exp-verbs." +#endif static inline int gds_transport_init() { int status = 0; if (!gds_main_transport) { gds_transport_t *t = NULL; + #if HAVE_EXP_VERBS status = gds_transport_mlx5_exp_init(&t); if (status) { gds_err("error in gds_transport_mlx5_exp_init\n"); goto out; } assert(t); + #else + status = ENOTSUPP; + goto out; + #endif gds_main_transport = t; } out: From 73404832fdd2b2bc6205b5b00470e7b55fe7c8be Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Sun, 17 Oct 2021 19:07:12 -0700 Subject: [PATCH 38/39] Fixed typo in configure.ac --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 34bc650..6f4449c 100644 --- a/configure.ac +++ b/configure.ac @@ -201,7 +201,7 @@ if test "x$have_exp_verbs" != "x" && test "x$have_peer_ops" != "x"; then AC_DEFINE([HAVE_EXP_VERBS], [1], [Define if exp-verbs exists.]) enable_exp_verbs=1 else - AC_MSG_WARN([This version of libgdsync cannot be used with out exp-verbs.]) + AC_MSG_WARN([This version of libgdsync cannot be used without exp-verbs.]) fi AM_CONDITIONAL([COMPILE_EXP_VERBS], [test "x$enable_exp_verbs" != "x"]) From 9986794f49cffc52864e33fcfd43669bb41493ac Mon Sep 17 00:00:00 2001 From: Pak Markthub Date: Sun, 17 Oct 2021 19:07:38 -0700 Subject: [PATCH 39/39] Fixed compile errors when compiling on x86 --- src/transport.hpp | 3 ++- src/utils.hpp | 1 + tests/gpu.h | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/transport.hpp b/src/transport.hpp index 00078cb..079f9ce 100644 --- a/src/transport.hpp +++ b/src/transport.hpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -77,7 +78,7 @@ static inline int gds_transport_init() } assert(t); #else - status = ENOTSUPP; + status = ENOTSUP; goto out; #endif gds_main_transport = t; diff --git a/src/utils.hpp b/src/utils.hpp index cd2102e..dccb125 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -31,6 +31,7 @@ #warning "__STDC_FORMAT_MACROS should be defined to pull definition of PRIx64, etc" #endif #include // to pull PRIx64 +#include // internal assert function diff --git a/tests/gpu.h b/tests/gpu.h index 0fde885..77a6db4 100644 --- a/tests/gpu.h +++ b/tests/gpu.h @@ -27,8 +27,18 @@ #pragma once +#include #include +#undef BEGIN_C_DECLS +#undef END_C_DECLS +#ifdef __cplusplus +# define BEGIN_C_DECLS extern "C" { +# define END_C_DECLS } +#else +# define BEGIN_C_DECLS +# define END_C_DECLS +#endif #ifdef USE_PROFILE #include