prov/efa: Add FI_MORE unit tests

prov/efa/Makefile.include

@@ -300,6 +300,8 @@ nodist_prov_efa_test_gtest_efa_gtest_SOURCES = \
 	prov/efa/test/gtest/efa_gtest_rdm_pke.cc \
 	prov/efa/test/gtest/efa_gtest_rdm_ope_helpers.c \
 	prov/efa/test/gtest/efa_gtest_rdm_ope.cc \
+	prov/efa/test/gtest/efa_gtest_fi_more_helpers.c \
+	prov/efa/test/gtest/efa_gtest_fi_more.cc \
 	prov/efa/test/gtest/efa_gtest_cq.cc \
 	prov/efa/test/gtest/efa_gtest_rdm_cntr.cc \
 	prov/efa/test/gtest/efa_gtest_rdm_peer.cc \
@@ -345,6 +347,7 @@ prov_efa_test_gtest_efa_gtest_LDFLAGS = \
 	-Wl,--wrap=efa_ibv_cq_wc_read_byte_len \
 	-Wl,--wrap=efa_ibv_cq_wc_read_src_qp \
 	-Wl,--wrap=efa_ibv_cq_wc_read_slid \
+	-Wl,--wrap=efa_ibv_cq_wc_is_unsolicited \
 	-Wl,--wrap=efa_rdm_pke_clone \
 	-Wl,--wrap=efa_qp_post_recv \
 	-Wl,--wrap=efa_qp_post_send \

prov/efa/src/efa_data_path_ops.h

@@ -30,11 +30,28 @@
 #include "efa_data_path_direct_entry.h"
 #endif
 
+/*
+ * EFA_PROD_STATIC_INLINE is defined to "static inline" in a production build
+ *
+ * Unit test build defines it to nothing, allowing the functions to be mocked
+ * A separate EFA_DATA_PATH_OPS_EMIT_BODIES macro is needed to pick out the
+ * TU that will emit the function bodies. Only one TU is allowed to define it.
+ * which is prov/efa/test/efa_unit_test_data_path_ops.c.
+ * Every other translation unit sees the declarations alone, so its calls are
+ * undefined references and are therefore wrappable.
+ */
+#if EFA_UNIT_TEST
+#define EFA_PROD_STATIC_INLINE
+#else
+#define EFA_PROD_STATIC_INLINE static inline
+#endif
+
+#if !EFA_UNIT_TEST || defined(EFA_DATA_PATH_OPS_EMIT_BODIES)
 
 /**
  * @brief RDMA-core version of send operation using ibv_* APIs
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_ibv_post_send(
 		struct efa_qp *qp,
 		const struct ibv_sge *sge_list,
@@ -84,7 +101,7 @@ efa_ibv_post_send(
 /**
  * @brief RDMA-core version of RDMA read operation using ibv_* APIs
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_ibv_post_read(
 		struct efa_qp *qp,
 		const struct ibv_sge *sge_list,
@@ -122,7 +139,7 @@ efa_ibv_post_read(
 /**
  * @brief RDMA-core version of RDMA write operation using ibv_* APIs
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_ibv_post_write(
 		struct efa_qp *qp,
 		const struct ibv_sge *sge_list,
@@ -176,9 +193,26 @@ efa_ibv_post_write(
 	return 0;
 }
 
+#endif /* bodies */
 
 #if EFA_UNIT_TEST
 /* For unit tests, declare functions that are defined in efa_unit_test_data_path_ops.c */
+int efa_ibv_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list,
+		      const struct ibv_data_buf *inline_data_list,
+		      size_t data_count, bool use_inline, uintptr_t wr_id,
+		      uint64_t data, uint64_t flags, struct efa_ah *ah,
+		      uint32_t qpn, uint32_t qkey);
+int efa_ibv_post_read(struct efa_qp *qp, const struct ibv_sge *sge_list,
+		      size_t sge_count, uint32_t remote_key,
+		      uint64_t remote_addr, uintptr_t wr_id, uint64_t flags,
+		      struct efa_ah *ah, uint32_t qpn, uint32_t qkey);
+int efa_ibv_post_write(struct efa_qp *qp, const struct ibv_sge *sge_list,
+		       size_t sge_count,
+		       const struct ibv_data_buf *inline_data_list,
+		       bool use_inline, uint32_t remote_key,
+		       uint64_t remote_addr, uintptr_t wr_id, uint64_t data,
+		       uint64_t flags, struct efa_ah *ah, uint32_t qpn,
+		       uint32_t qkey);
 int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad);
 int efa_qp_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list,
 		      const struct ibv_data_buf *inline_data_list,
@@ -213,11 +247,12 @@ int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid);
 int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context);
 int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only);
 
-#else
-/* For production, define static inline functions */
+#endif /* EFA_UNIT_TEST */
+
+#if !EFA_UNIT_TEST || defined(EFA_DATA_PATH_OPS_EMIT_BODIES)
 
 /* QP wrapper functions */
-static inline int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad)
+EFA_PROD_STATIC_INLINE int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (qp->data_path_direct_enabled)
@@ -229,7 +264,7 @@ static inline int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, st
 /**
  * @brief Wrapper for send operations - chooses between direct and IBV paths
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_qp_post_send(struct efa_qp *qp,
                  const struct ibv_sge *sge_list,
                  const struct ibv_data_buf *inline_data_list,
@@ -256,7 +291,7 @@ efa_qp_post_send(struct efa_qp *qp,
 /**
  * @brief Wrapper for RDMA read operations - chooses between direct and IBV paths
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_qp_post_read(struct efa_qp *qp,
                  const struct ibv_sge *sge_list,
                  size_t sge_count,
@@ -282,7 +317,7 @@ efa_qp_post_read(struct efa_qp *qp,
 /**
  * @brief Wrapper for RDMA write operations - chooses between direct and IBV paths
  */
-static inline int
+EFA_PROD_STATIC_INLINE int
 efa_qp_post_write(struct efa_qp *qp,
                   const struct ibv_sge *sge_list,
                   size_t sge_count,
@@ -311,7 +346,7 @@ efa_qp_post_write(struct efa_qp *qp,
 }
 
 /* CQ wrapper functions */
-static inline int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr)
+EFA_PROD_STATIC_INLINE int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -320,7 +355,7 @@ static inline int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_po
 	return ibv_start_poll(ibv_cq->ibv_cq_ex, attr);
 }
 
-static inline int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -329,7 +364,7 @@ static inline int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq)
 	return ibv_next_poll(ibv_cq->ibv_cq_ex);
 }
 
-static inline enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -338,7 +373,7 @@ static inline enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ib
 	return ibv_wc_read_opcode(ibv_cq->ibv_cq_ex);
 }
 
-static inline void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled) {
@@ -349,7 +384,7 @@ static inline void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq)
 	ibv_end_poll(ibv_cq->ibv_cq_ex);
 }
 
-static inline uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -358,7 +393,7 @@ static inline uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq)
 	return ibv_wc_read_qp_num(ibv_cq->ibv_cq_ex);
 }
 
-static inline uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -367,7 +402,7 @@ static inline uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq)
 	return ibv_wc_read_vendor_err(ibv_cq->ibv_cq_ex);
 }
 
-static inline uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -376,7 +411,7 @@ static inline uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq)
 	return ibv_wc_read_src_qp(ibv_cq->ibv_cq_ex);
 }
 
-static inline uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -385,7 +420,7 @@ static inline uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq)
 	return ibv_wc_read_slid(ibv_cq->ibv_cq_ex);
 }
 
-static inline uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -394,7 +429,7 @@ static inline uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq)
 	return ibv_wc_read_byte_len(ibv_cq->ibv_cq_ex);
 }
 
-static inline unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -403,7 +438,7 @@ static inline unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq
 	return ibv_wc_read_wc_flags(ibv_cq->ibv_cq_ex);
 }
 
-static inline __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -413,7 +448,7 @@ static inline __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq)
 }
 
 
-static inline bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq)
+EFA_PROD_STATIC_INLINE bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -426,7 +461,7 @@ static inline bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq)
 #endif
 }
 
-static inline int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid)
+EFA_PROD_STATIC_INLINE int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	if (ibv_cq->data_path_direct_enabled)
@@ -440,7 +475,7 @@ static inline int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_g
 #endif
 }
 
-static inline int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context)
+EFA_PROD_STATIC_INLINE int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context)
 {
 	struct ibv_cq *cq = ibv_cq_ex_to_cq(ibv_cq->ibv_cq_ex);
 #if HAVE_EFA_DATA_PATH_DIRECT && HAVE_EFADV_CQ_ATTR_DB
@@ -454,7 +489,7 @@ static inline int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_cont
 #endif
 }
 
-static inline int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only)
+EFA_PROD_STATIC_INLINE int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only)
 {
 #if HAVE_EFA_DATA_PATH_DIRECT && HAVE_EFADV_CQ_ATTR_DB
 	if (ibv_cq->data_path_direct_enabled)
@@ -468,7 +503,7 @@ static inline int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited
 }
 
 
-#endif /* EFA_UNIT_TEST */
+#endif /* bodies */
 
 /**
  * @brief Check whether a completion consumes recv buffer

prov/efa/src/rdm/efa_rdm_pke.c

@@ -615,6 +615,7 @@ int efa_rdm_pke_read(struct efa_rdm_pke *pkt_entry,
 	struct efa_ah *ah;
 	uint32_t qpn, qkey;
 	uint64_t wr_id;
+	uint64_t flags = 0;
 
 	ep = pkt_entry->ep;
 	assert(ep);
@@ -640,7 +641,15 @@ int efa_rdm_pke_read(struct efa_rdm_pke *pkt_entry,
 
 	wr_id = efa_rdm_pke_get_wr_id(pkt_entry);
 
-	err = efa_qp_post_read(qp, &sge, 1, remote_key, remote_buf, wr_id, 0,
+	/*
+	 * A read posted for an rxe belongs to a receive-side protocol such as
+	 * longread. Such a read request should not have FI_MORE set.
+	 */
+	assert(txe->type == EFA_RDM_TXE || !(txe->fi_flags & FI_MORE));
+	if (txe->fi_flags & FI_MORE)
+        flags |= FI_MORE;
+
+	err = efa_qp_post_read(qp, &sge, 1, remote_key, remote_buf, wr_id, flags,
 			       ah, qpn, qkey);
 
 #if ENABLE_DEBUG

prov/efa/src/rdm/efa_rdm_srx.c

@@ -21,7 +21,12 @@ void efa_rdm_srx_update_rxe(struct fi_peer_rx_entry *peer_rxe,
 {
 	assert(peer_rxe->count <= rxe->ep->base_ep.info->rx_attr->iov_limit);
 
-	rxe->fi_flags = peer_rxe->flags;
+	/*
+	 * Do not set FI_MORE for RXEs. The protocol path posts bounce buffers for
+	 * the receive, so it does not make sense to honor FI_MORE. Leaving it set
+	 * could set FI_MORE on a read issued during long read and cause a hang.
+	 */
+	rxe->fi_flags = peer_rxe->flags & ~FI_MORE;
 
 	/* Handle case where we're allocating an unexpected rxe */
 	rxe->iov_count = peer_rxe->count;