diff --git a/prov/efa/Makefile.include b/prov/efa/Makefile.include index 6f23f3353a3..1b51027bdad 100644 --- a/prov/efa/Makefile.include +++ b/prov/efa/Makefile.include @@ -300,6 +300,8 @@ nodist_prov_efa_test_gtest_efa_gtest_SOURCES = \ prov/efa/test/gtest/efa_gtest_rdm_pke.cc \ prov/efa/test/gtest/efa_gtest_rdm_ope_helpers.c \ prov/efa/test/gtest/efa_gtest_rdm_ope.cc \ + prov/efa/test/gtest/efa_gtest_fi_more_helpers.c \ + prov/efa/test/gtest/efa_gtest_fi_more.cc \ prov/efa/test/gtest/efa_gtest_cq.cc \ prov/efa/test/gtest/efa_gtest_rdm_cntr.cc \ prov/efa/test/gtest/efa_gtest_rdm_peer.cc \ @@ -345,6 +347,7 @@ prov_efa_test_gtest_efa_gtest_LDFLAGS = \ -Wl,--wrap=efa_ibv_cq_wc_read_byte_len \ -Wl,--wrap=efa_ibv_cq_wc_read_src_qp \ -Wl,--wrap=efa_ibv_cq_wc_read_slid \ + -Wl,--wrap=efa_ibv_cq_wc_is_unsolicited \ -Wl,--wrap=efa_rdm_pke_clone \ -Wl,--wrap=efa_qp_post_recv \ -Wl,--wrap=efa_qp_post_send \ diff --git a/prov/efa/src/efa_data_path_ops.h b/prov/efa/src/efa_data_path_ops.h index 9c2646075f2..116e1c6ad63 100644 --- a/prov/efa/src/efa_data_path_ops.h +++ b/prov/efa/src/efa_data_path_ops.h @@ -30,11 +30,28 @@ #include "efa_data_path_direct_entry.h" #endif +/* + * EFA_PROD_STATIC_INLINE is defined to "static inline" in a production build + * + * Unit test build defines it to nothing, allowing the functions to be mocked + * A separate EFA_DATA_PATH_OPS_EMIT_BODIES macro is needed to pick out the + * TU that will emit the function bodies. Only one TU is allowed to define it. + * which is prov/efa/test/efa_unit_test_data_path_ops.c. + * Every other translation unit sees the declarations alone, so its calls are + * undefined references and are therefore wrappable. + */ +#if EFA_UNIT_TEST +#define EFA_PROD_STATIC_INLINE +#else +#define EFA_PROD_STATIC_INLINE static inline +#endif + +#if !EFA_UNIT_TEST || defined(EFA_DATA_PATH_OPS_EMIT_BODIES) /** * @brief RDMA-core version of send operation using ibv_* APIs */ -static inline int +EFA_PROD_STATIC_INLINE int efa_ibv_post_send( struct efa_qp *qp, const struct ibv_sge *sge_list, @@ -84,7 +101,7 @@ efa_ibv_post_send( /** * @brief RDMA-core version of RDMA read operation using ibv_* APIs */ -static inline int +EFA_PROD_STATIC_INLINE int efa_ibv_post_read( struct efa_qp *qp, const struct ibv_sge *sge_list, @@ -122,7 +139,7 @@ efa_ibv_post_read( /** * @brief RDMA-core version of RDMA write operation using ibv_* APIs */ -static inline int +EFA_PROD_STATIC_INLINE int efa_ibv_post_write( struct efa_qp *qp, const struct ibv_sge *sge_list, @@ -176,9 +193,26 @@ efa_ibv_post_write( return 0; } +#endif /* bodies */ #if EFA_UNIT_TEST /* For unit tests, declare functions that are defined in efa_unit_test_data_path_ops.c */ +int efa_ibv_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list, + const struct ibv_data_buf *inline_data_list, + size_t data_count, bool use_inline, uintptr_t wr_id, + uint64_t data, uint64_t flags, struct efa_ah *ah, + uint32_t qpn, uint32_t qkey); +int efa_ibv_post_read(struct efa_qp *qp, const struct ibv_sge *sge_list, + size_t sge_count, uint32_t remote_key, + uint64_t remote_addr, uintptr_t wr_id, uint64_t flags, + struct efa_ah *ah, uint32_t qpn, uint32_t qkey); +int efa_ibv_post_write(struct efa_qp *qp, const struct ibv_sge *sge_list, + size_t sge_count, + const struct ibv_data_buf *inline_data_list, + bool use_inline, uint32_t remote_key, + uint64_t remote_addr, uintptr_t wr_id, uint64_t data, + uint64_t flags, struct efa_ah *ah, uint32_t qpn, + uint32_t qkey); int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad); int efa_qp_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list, const struct ibv_data_buf *inline_data_list, @@ -213,11 +247,12 @@ int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid); int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context); int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only); -#else -/* For production, define static inline functions */ +#endif /* EFA_UNIT_TEST */ + +#if !EFA_UNIT_TEST || defined(EFA_DATA_PATH_OPS_EMIT_BODIES) /* QP wrapper functions */ -static inline int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) +EFA_PROD_STATIC_INLINE int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) { #if HAVE_EFA_DATA_PATH_DIRECT if (qp->data_path_direct_enabled) @@ -229,7 +264,7 @@ static inline int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, st /** * @brief Wrapper for send operations - chooses between direct and IBV paths */ -static inline int +EFA_PROD_STATIC_INLINE int efa_qp_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list, const struct ibv_data_buf *inline_data_list, @@ -256,7 +291,7 @@ efa_qp_post_send(struct efa_qp *qp, /** * @brief Wrapper for RDMA read operations - chooses between direct and IBV paths */ -static inline int +EFA_PROD_STATIC_INLINE int efa_qp_post_read(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, @@ -282,7 +317,7 @@ efa_qp_post_read(struct efa_qp *qp, /** * @brief Wrapper for RDMA write operations - chooses between direct and IBV paths */ -static inline int +EFA_PROD_STATIC_INLINE int efa_qp_post_write(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, @@ -311,7 +346,7 @@ efa_qp_post_write(struct efa_qp *qp, } /* CQ wrapper functions */ -static inline int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr) +EFA_PROD_STATIC_INLINE int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -320,7 +355,7 @@ static inline int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_po return ibv_start_poll(ibv_cq->ibv_cq_ex, attr); } -static inline int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -329,7 +364,7 @@ static inline int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq) return ibv_next_poll(ibv_cq->ibv_cq_ex); } -static inline enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -338,7 +373,7 @@ static inline enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ib return ibv_wc_read_opcode(ibv_cq->ibv_cq_ex); } -static inline void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) { @@ -349,7 +384,7 @@ static inline void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq) ibv_end_poll(ibv_cq->ibv_cq_ex); } -static inline uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -358,7 +393,7 @@ static inline uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq) return ibv_wc_read_qp_num(ibv_cq->ibv_cq_ex); } -static inline uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -367,7 +402,7 @@ static inline uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq) return ibv_wc_read_vendor_err(ibv_cq->ibv_cq_ex); } -static inline uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -376,7 +411,7 @@ static inline uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq) return ibv_wc_read_src_qp(ibv_cq->ibv_cq_ex); } -static inline uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -385,7 +420,7 @@ static inline uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq) return ibv_wc_read_slid(ibv_cq->ibv_cq_ex); } -static inline uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -394,7 +429,7 @@ static inline uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq) return ibv_wc_read_byte_len(ibv_cq->ibv_cq_ex); } -static inline unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -403,7 +438,7 @@ static inline unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq return ibv_wc_read_wc_flags(ibv_cq->ibv_cq_ex); } -static inline __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -413,7 +448,7 @@ static inline __be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq) } -static inline bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq) +EFA_PROD_STATIC_INLINE bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -426,7 +461,7 @@ static inline bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq) #endif } -static inline int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid) +EFA_PROD_STATIC_INLINE int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid) { #if HAVE_EFA_DATA_PATH_DIRECT if (ibv_cq->data_path_direct_enabled) @@ -440,7 +475,7 @@ static inline int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_g #endif } -static inline int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context) +EFA_PROD_STATIC_INLINE int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context) { struct ibv_cq *cq = ibv_cq_ex_to_cq(ibv_cq->ibv_cq_ex); #if HAVE_EFA_DATA_PATH_DIRECT && HAVE_EFADV_CQ_ATTR_DB @@ -454,7 +489,7 @@ static inline int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_cont #endif } -static inline int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only) +EFA_PROD_STATIC_INLINE int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only) { #if HAVE_EFA_DATA_PATH_DIRECT && HAVE_EFADV_CQ_ATTR_DB if (ibv_cq->data_path_direct_enabled) @@ -468,7 +503,7 @@ static inline int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited } -#endif /* EFA_UNIT_TEST */ +#endif /* bodies */ /** * @brief Check whether a completion consumes recv buffer diff --git a/prov/efa/src/rdm/efa_rdm_pke.c b/prov/efa/src/rdm/efa_rdm_pke.c index 06835d648a9..f12bae230f1 100644 --- a/prov/efa/src/rdm/efa_rdm_pke.c +++ b/prov/efa/src/rdm/efa_rdm_pke.c @@ -615,6 +615,7 @@ int efa_rdm_pke_read(struct efa_rdm_pke *pkt_entry, struct efa_ah *ah; uint32_t qpn, qkey; uint64_t wr_id; + uint64_t flags = 0; ep = pkt_entry->ep; assert(ep); @@ -640,7 +641,15 @@ int efa_rdm_pke_read(struct efa_rdm_pke *pkt_entry, wr_id = efa_rdm_pke_get_wr_id(pkt_entry); - err = efa_qp_post_read(qp, &sge, 1, remote_key, remote_buf, wr_id, 0, + /* + * A read posted for an rxe belongs to a receive-side protocol such as + * longread. Such a read request should not have FI_MORE set. + */ + assert(txe->type == EFA_RDM_TXE || !(txe->fi_flags & FI_MORE)); + if (txe->fi_flags & FI_MORE) + flags |= FI_MORE; + + err = efa_qp_post_read(qp, &sge, 1, remote_key, remote_buf, wr_id, flags, ah, qpn, qkey); #if ENABLE_DEBUG diff --git a/prov/efa/src/rdm/efa_rdm_srx.c b/prov/efa/src/rdm/efa_rdm_srx.c index f2afe06e478..eb04b070755 100644 --- a/prov/efa/src/rdm/efa_rdm_srx.c +++ b/prov/efa/src/rdm/efa_rdm_srx.c @@ -21,7 +21,12 @@ void efa_rdm_srx_update_rxe(struct fi_peer_rx_entry *peer_rxe, { assert(peer_rxe->count <= rxe->ep->base_ep.info->rx_attr->iov_limit); - rxe->fi_flags = peer_rxe->flags; + /* + * Do not set FI_MORE for RXEs. The protocol path posts bounce buffers for + * the receive, so it does not make sense to honor FI_MORE. Leaving it set + * could set FI_MORE on a read issued during long read and cause a hang. + */ + rxe->fi_flags = peer_rxe->flags & ~FI_MORE; /* Handle case where we're allocating an unexpected rxe */ rxe->iov_count = peer_rxe->count; diff --git a/prov/efa/test/efa_unit_test_data_path_ops.c b/prov/efa/test/efa_unit_test_data_path_ops.c index d18f1c5bd18..cfd30fc36ca 100644 --- a/prov/efa/test/efa_unit_test_data_path_ops.c +++ b/prov/efa/test/efa_unit_test_data_path_ops.c @@ -2,122 +2,20 @@ /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */ /* - * Unit test stub versions of EFA data path operations + * efa_data_path_ops.h gives these functions external linkage under + * EFA_UNIT_TEST rather than defining them static inline, so that they can be + * mocked. Only one translation unit may define them by defining the macro + * EFA_DATA_PATH_OPS_EMIT_BODIES. This TU defines EFA_DATA_PATH_OPS_EMIT_BODIES * - * This file provides stub implementations of the data path operations - * specifically for unit testing. These functions are needed because - * static inline functions cannot have their addresses taken for - * function pointers in the unit test mock structure. + * The macro _has_ to be defined before the first include that could reach + * efa_data_path_ops.h. Otherwise, linking will fail with undefined references + * to efa_qp_post_*. */ +#define EFA_DATA_PATH_OPS_EMIT_BODIES + #include #include #include "efa_cq.h" #include "efa_base_ep.h" - -/* QP wrapper functions - unit test stubs */ -int efa_qp_post_recv(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) -{ - return 0; -} - -int efa_qp_post_send(struct efa_qp *qp, const struct ibv_sge *sge_list, - const struct ibv_data_buf *inline_data_list, size_t iov_count, - bool use_inline, uintptr_t wr_id, uint64_t data, uint64_t flags, - struct efa_ah *ah, uint32_t qpn, uint32_t qkey) -{ - return 0; -} - -int efa_qp_post_read(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, - uint32_t remote_key, uint64_t remote_addr, uintptr_t wr_id, - uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey) -{ - return 0; -} - -int efa_qp_post_write(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, - const struct ibv_data_buf *inline_data_list, bool use_inline, - uint32_t remote_key, uint64_t remote_addr, uintptr_t wr_id, - uint64_t data, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey) -{ - return 0; -} - - - -/* CQ wrapper functions - unit test stubs */ -int efa_ibv_cq_start_poll(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr) -{ - return ENOENT; -} - -int efa_ibv_cq_next_poll(struct efa_ibv_cq *ibv_cq) -{ - return ENOENT; -} - -enum ibv_wc_opcode efa_ibv_cq_wc_read_opcode(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -void efa_ibv_cq_end_poll(struct efa_ibv_cq *ibv_cq) -{ -} - -uint32_t efa_ibv_cq_wc_read_qp_num(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -uint32_t efa_ibv_cq_wc_read_vendor_err(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -uint32_t efa_ibv_cq_wc_read_src_qp(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -uint32_t efa_ibv_cq_wc_read_slid(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -uint32_t efa_ibv_cq_wc_read_byte_len(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -unsigned int efa_ibv_cq_wc_read_wc_flags(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -__be32 efa_ibv_cq_wc_read_imm_data(struct efa_ibv_cq *ibv_cq) -{ - return 0; -} - -bool efa_ibv_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq) -{ - return false; -} - - -int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid) -{ - return ENOSYS; -} - -int efa_ibv_get_cq_event(struct efa_ibv_cq *ibv_cq, void **cq_context) -{ - return 0; -} - -int efa_ibv_req_notify_cq(struct efa_ibv_cq *ibv_cq, int solicited_only) -{ - return 0; -} +#include "efa_data_path_ops.h" diff --git a/prov/efa/test/efa_unit_test_mocks.c b/prov/efa/test/efa_unit_test_mocks.c index b60aa0b9129..cda376d155e 100644 --- a/prov/efa/test/efa_unit_test_mocks.c +++ b/prov/efa/test/efa_unit_test_mocks.c @@ -387,6 +387,115 @@ int efa_mock_efa_qp_post_send_verify_handshake_pkt_local_host_id_and_save_wr(str return mock_int(); } +/* + * Inert data path mocks: the defaults installed by the group setup, since an + * unmocked data path op would now reach the real device. + */ +int efa_mock_efa_qp_post_recv_no_op(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) +{ + return 0; +} + +int efa_mock_efa_qp_post_send_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, + const struct ibv_data_buf *inline_data_list, size_t iov_count, + bool use_inline, uintptr_t wr_id, uint64_t data, uint64_t flags, + struct efa_ah *ah, uint32_t qpn, uint32_t qkey) +{ + return 0; +} + +int efa_mock_efa_qp_post_read_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, + size_t sge_count, uint32_t remote_key, uint64_t remote_addr, + uintptr_t wr_id, uint64_t flags, struct efa_ah *ah, + uint32_t qpn, uint32_t qkey) +{ + return 0; +} + +int efa_mock_efa_qp_post_write_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, + size_t sge_count, const struct ibv_data_buf *inline_data_list, + bool use_inline, uint32_t remote_key, uint64_t remote_addr, + uintptr_t wr_id, uint64_t data, uint64_t flags, + struct efa_ah *ah, uint32_t qpn, uint32_t qkey) +{ + return 0; +} + +/* Report no completion, so nothing in a test binary ever reaps a CQE. */ +int efa_mock_efa_ibv_cq_start_poll_no_cqe(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr) +{ + return ENOENT; +} + +int efa_mock_efa_ibv_cq_next_poll_no_cqe(struct efa_ibv_cq *ibv_cq) +{ + return ENOENT; +} + +enum ibv_wc_opcode efa_mock_efa_ibv_cq_wc_read_opcode_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +void efa_mock_efa_ibv_cq_end_poll_no_op(struct efa_ibv_cq *ibv_cq) +{ +} + +uint32_t efa_mock_efa_ibv_cq_wc_read_qp_num_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +uint32_t efa_mock_efa_ibv_cq_wc_read_vendor_err_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +uint32_t efa_mock_efa_ibv_cq_wc_read_src_qp_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +uint32_t efa_mock_efa_ibv_cq_wc_read_slid_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +uint32_t efa_mock_efa_ibv_cq_wc_read_byte_len_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +unsigned int efa_mock_efa_ibv_cq_wc_read_wc_flags_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +__be32 efa_mock_efa_ibv_cq_wc_read_imm_data_no_op(struct efa_ibv_cq *ibv_cq) +{ + return 0; +} + +bool efa_mock_efa_ibv_cq_wc_is_unsolicited_no_op(struct efa_ibv_cq *ibv_cq) +{ + return false; +} + +int efa_mock_efa_ibv_cq_wc_read_sgid_no_op(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid) +{ + return ENOSYS; +} + +int efa_mock_efa_ibv_get_cq_event_no_op(struct efa_ibv_cq *ibv_cq, void **cq_context) +{ + return 0; +} + +int efa_mock_efa_ibv_req_notify_cq_no_op(struct efa_ibv_cq *ibv_cq, int solicited_only) +{ + return 0; +} + struct efa_unit_test_mocks g_efa_unit_test_mocks = { .local_host_id = 0, .peer_host_id = 0, @@ -419,25 +528,25 @@ struct efa_unit_test_mocks g_efa_unit_test_mocks = { .ibv_is_fork_initialized = __real_ibv_is_fork_initialized, /* EFA data path ops real function assignments */ - .efa_qp_post_recv = __real_efa_qp_post_recv, - .efa_qp_post_send = __real_efa_qp_post_send, - .efa_qp_post_read = __real_efa_qp_post_read, - .efa_qp_post_write = __real_efa_qp_post_write, - .efa_ibv_cq_start_poll = __real_efa_ibv_cq_start_poll, - .efa_ibv_cq_next_poll = __real_efa_ibv_cq_next_poll, - .efa_ibv_cq_wc_read_opcode = __real_efa_ibv_cq_wc_read_opcode, - .efa_ibv_cq_end_poll = __real_efa_ibv_cq_end_poll, - .efa_ibv_cq_wc_read_qp_num = __real_efa_ibv_cq_wc_read_qp_num, - .efa_ibv_cq_wc_read_vendor_err = __real_efa_ibv_cq_wc_read_vendor_err, - .efa_ibv_cq_wc_read_src_qp = __real_efa_ibv_cq_wc_read_src_qp, - .efa_ibv_cq_wc_read_slid = __real_efa_ibv_cq_wc_read_slid, - .efa_ibv_cq_wc_read_byte_len = __real_efa_ibv_cq_wc_read_byte_len, - .efa_ibv_cq_wc_read_wc_flags = __real_efa_ibv_cq_wc_read_wc_flags, - .efa_ibv_cq_wc_read_imm_data = __real_efa_ibv_cq_wc_read_imm_data, - .efa_ibv_cq_wc_is_unsolicited = __real_efa_ibv_cq_wc_is_unsolicited, - .efa_ibv_cq_wc_read_sgid = __real_efa_ibv_cq_wc_read_sgid, - .efa_ibv_get_cq_event = __real_efa_ibv_get_cq_event, - .efa_ibv_req_notify_cq = __real_efa_ibv_req_notify_cq, + .efa_qp_post_recv = efa_mock_efa_qp_post_recv_no_op, + .efa_qp_post_send = efa_mock_efa_qp_post_send_no_op, + .efa_qp_post_read = efa_mock_efa_qp_post_read_no_op, + .efa_qp_post_write = efa_mock_efa_qp_post_write_no_op, + .efa_ibv_cq_start_poll = efa_mock_efa_ibv_cq_start_poll_no_cqe, + .efa_ibv_cq_next_poll = efa_mock_efa_ibv_cq_next_poll_no_cqe, + .efa_ibv_cq_wc_read_opcode = efa_mock_efa_ibv_cq_wc_read_opcode_no_op, + .efa_ibv_cq_end_poll = efa_mock_efa_ibv_cq_end_poll_no_op, + .efa_ibv_cq_wc_read_qp_num = efa_mock_efa_ibv_cq_wc_read_qp_num_no_op, + .efa_ibv_cq_wc_read_vendor_err = efa_mock_efa_ibv_cq_wc_read_vendor_err_no_op, + .efa_ibv_cq_wc_read_src_qp = efa_mock_efa_ibv_cq_wc_read_src_qp_no_op, + .efa_ibv_cq_wc_read_slid = efa_mock_efa_ibv_cq_wc_read_slid_no_op, + .efa_ibv_cq_wc_read_byte_len = efa_mock_efa_ibv_cq_wc_read_byte_len_no_op, + .efa_ibv_cq_wc_read_wc_flags = efa_mock_efa_ibv_cq_wc_read_wc_flags_no_op, + .efa_ibv_cq_wc_read_imm_data = efa_mock_efa_ibv_cq_wc_read_imm_data_no_op, + .efa_ibv_cq_wc_is_unsolicited = efa_mock_efa_ibv_cq_wc_is_unsolicited_no_op, + .efa_ibv_cq_wc_read_sgid = efa_mock_efa_ibv_cq_wc_read_sgid_no_op, + .efa_ibv_get_cq_event = efa_mock_efa_ibv_get_cq_event_no_op, + .efa_ibv_req_notify_cq = efa_mock_efa_ibv_req_notify_cq_no_op, #if HAVE_EFADV_QUERY_MR .efadv_query_mr = __real_efadv_query_mr, diff --git a/prov/efa/test/efa_unit_test_mocks.h b/prov/efa/test/efa_unit_test_mocks.h index c18b4c91338..5ea1ff5cca2 100644 --- a/prov/efa/test/efa_unit_test_mocks.h +++ b/prov/efa/test/efa_unit_test_mocks.h @@ -136,6 +136,32 @@ int efa_mock_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); /* EFA data path ops mock helpers */ +/* + * Inert data path mocks. The data path ops now reach the real device when + * unmocked, so the mock table defaults to these: they report success without + * submitting anything, and no completion at all. A test that wants the device + * installs __real_* explicitly. + */ +int efa_mock_efa_qp_post_recv_no_op(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad); +int efa_mock_efa_qp_post_send_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, const struct ibv_data_buf *inline_data_list, size_t iov_count, bool use_inline, uintptr_t wr_id, uint64_t data, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey); +int efa_mock_efa_qp_post_read_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, uint32_t remote_key, uint64_t remote_addr, uintptr_t wr_id, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey); +int efa_mock_efa_qp_post_write_no_op(struct efa_qp *qp, const struct ibv_sge *sge_list, size_t sge_count, const struct ibv_data_buf *inline_data_list, bool use_inline, uint32_t remote_key, uint64_t remote_addr, uintptr_t wr_id, uint64_t data, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey); +int efa_mock_efa_ibv_cq_start_poll_no_cqe(struct efa_ibv_cq *ibv_cq, struct ibv_poll_cq_attr *attr); +int efa_mock_efa_ibv_cq_next_poll_no_cqe(struct efa_ibv_cq *ibv_cq); +enum ibv_wc_opcode efa_mock_efa_ibv_cq_wc_read_opcode_no_op(struct efa_ibv_cq *ibv_cq); +void efa_mock_efa_ibv_cq_end_poll_no_op(struct efa_ibv_cq *ibv_cq); +uint32_t efa_mock_efa_ibv_cq_wc_read_qp_num_no_op(struct efa_ibv_cq *ibv_cq); +uint32_t efa_mock_efa_ibv_cq_wc_read_vendor_err_no_op(struct efa_ibv_cq *ibv_cq); +uint32_t efa_mock_efa_ibv_cq_wc_read_src_qp_no_op(struct efa_ibv_cq *ibv_cq); +uint32_t efa_mock_efa_ibv_cq_wc_read_slid_no_op(struct efa_ibv_cq *ibv_cq); +uint32_t efa_mock_efa_ibv_cq_wc_read_byte_len_no_op(struct efa_ibv_cq *ibv_cq); +unsigned int efa_mock_efa_ibv_cq_wc_read_wc_flags_no_op(struct efa_ibv_cq *ibv_cq); +__be32 efa_mock_efa_ibv_cq_wc_read_imm_data_no_op(struct efa_ibv_cq *ibv_cq); +bool efa_mock_efa_ibv_cq_wc_is_unsolicited_no_op(struct efa_ibv_cq *ibv_cq); +int efa_mock_efa_ibv_cq_wc_read_sgid_no_op(struct efa_ibv_cq *ibv_cq, union ibv_gid *sgid); +int efa_mock_efa_ibv_get_cq_event_no_op(struct efa_ibv_cq *ibv_cq, void **cq_context); +int efa_mock_efa_ibv_req_notify_cq_no_op(struct efa_ibv_cq *ibv_cq, int solicited_only); + int efa_mock_efa_qp_post_recv_return_mock(struct efa_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad); int efa_mock_efa_qp_post_send_return_mock(struct efa_qp *qp, const struct ibv_sge *sge_list, const struct ibv_data_buf *inline_data_list, size_t iov_count, bool use_inline, uintptr_t wr_id, uint64_t data, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey); int efa_mock_efa_qp_post_send_verify_not_inline(struct efa_qp *qp, const struct ibv_sge *sge_list, const struct ibv_data_buf *inline_data_list, size_t iov_count, bool use_inline, uintptr_t wr_id, uint64_t data, uint64_t flags, struct efa_ah *ah, uint32_t qpn, uint32_t qkey); diff --git a/prov/efa/test/efa_unit_tests.c b/prov/efa/test/efa_unit_tests.c index f298f9330d7..dbba0df503a 100644 --- a/prov/efa/test/efa_unit_tests.c +++ b/prov/efa/test/efa_unit_tests.c @@ -103,25 +103,25 @@ static int efa_unit_test_mocks_teardown(void **state) .ibv_is_fork_initialized = __real_ibv_is_fork_initialized, /* EFA data path ops real function assignments */ - .efa_qp_post_recv = __real_efa_qp_post_recv, - .efa_qp_post_send = __real_efa_qp_post_send, - .efa_qp_post_read = __real_efa_qp_post_read, - .efa_qp_post_write = __real_efa_qp_post_write, - .efa_ibv_cq_start_poll = __real_efa_ibv_cq_start_poll, - .efa_ibv_cq_next_poll = __real_efa_ibv_cq_next_poll, - .efa_ibv_cq_wc_read_opcode = __real_efa_ibv_cq_wc_read_opcode, - .efa_ibv_cq_end_poll = __real_efa_ibv_cq_end_poll, - .efa_ibv_cq_wc_read_qp_num = __real_efa_ibv_cq_wc_read_qp_num, - .efa_ibv_cq_wc_read_vendor_err = __real_efa_ibv_cq_wc_read_vendor_err, - .efa_ibv_cq_wc_read_src_qp = __real_efa_ibv_cq_wc_read_src_qp, - .efa_ibv_cq_wc_read_slid = __real_efa_ibv_cq_wc_read_slid, - .efa_ibv_cq_wc_read_byte_len = __real_efa_ibv_cq_wc_read_byte_len, - .efa_ibv_cq_wc_read_wc_flags = __real_efa_ibv_cq_wc_read_wc_flags, - .efa_ibv_cq_wc_read_imm_data = __real_efa_ibv_cq_wc_read_imm_data, - .efa_ibv_cq_wc_is_unsolicited = __real_efa_ibv_cq_wc_is_unsolicited, - .efa_ibv_cq_wc_read_sgid = __real_efa_ibv_cq_wc_read_sgid, - .efa_ibv_get_cq_event = __real_efa_ibv_get_cq_event, - .efa_ibv_req_notify_cq = __real_efa_ibv_req_notify_cq, + .efa_qp_post_recv = efa_mock_efa_qp_post_recv_no_op, + .efa_qp_post_send = efa_mock_efa_qp_post_send_no_op, + .efa_qp_post_read = efa_mock_efa_qp_post_read_no_op, + .efa_qp_post_write = efa_mock_efa_qp_post_write_no_op, + .efa_ibv_cq_start_poll = efa_mock_efa_ibv_cq_start_poll_no_cqe, + .efa_ibv_cq_next_poll = efa_mock_efa_ibv_cq_next_poll_no_cqe, + .efa_ibv_cq_wc_read_opcode = efa_mock_efa_ibv_cq_wc_read_opcode_no_op, + .efa_ibv_cq_end_poll = efa_mock_efa_ibv_cq_end_poll_no_op, + .efa_ibv_cq_wc_read_qp_num = efa_mock_efa_ibv_cq_wc_read_qp_num_no_op, + .efa_ibv_cq_wc_read_vendor_err = efa_mock_efa_ibv_cq_wc_read_vendor_err_no_op, + .efa_ibv_cq_wc_read_src_qp = efa_mock_efa_ibv_cq_wc_read_src_qp_no_op, + .efa_ibv_cq_wc_read_slid = efa_mock_efa_ibv_cq_wc_read_slid_no_op, + .efa_ibv_cq_wc_read_byte_len = efa_mock_efa_ibv_cq_wc_read_byte_len_no_op, + .efa_ibv_cq_wc_read_wc_flags = efa_mock_efa_ibv_cq_wc_read_wc_flags_no_op, + .efa_ibv_cq_wc_read_imm_data = efa_mock_efa_ibv_cq_wc_read_imm_data_no_op, + .efa_ibv_cq_wc_is_unsolicited = efa_mock_efa_ibv_cq_wc_is_unsolicited_no_op, + .efa_ibv_cq_wc_read_sgid = efa_mock_efa_ibv_cq_wc_read_sgid_no_op, + .efa_ibv_get_cq_event = efa_mock_efa_ibv_get_cq_event_no_op, + .efa_ibv_req_notify_cq = efa_mock_efa_ibv_req_notify_cq_no_op, #if HAVE_EFADV_QUERY_MR .efadv_query_mr = __real_efadv_query_mr, diff --git a/prov/efa/test/gtest/AGENTS.md b/prov/efa/test/gtest/AGENTS.md index a056fa77fe8..10763d75f38 100644 --- a/prov/efa/test/gtest/AGENTS.md +++ b/prov/efa/test/gtest/AGENTS.md @@ -74,6 +74,16 @@ contract or copy a pattern from a nearby test when the docs can settle it. `__real_`. Arming is what confines a `--wrap` (a process-wide symbol) to the test that cares. No-arg form matches any args; trailing args are matchers (`EFA_EXPECT_CALL(mock, ibv_destroy_ah, &ah)` → `ibv_destroy_ah(&ah)`). +- **`--wrap` cannot intercept a call between two functions in the same object + file.** The linker only rewrites *undefined external references*, so a call the + compiler resolved locally is invisible to it. A mockable callee therefore has to + live in a different translation unit from its callers: the `efa_cq_*` helpers + stay `static inline` and inline into `efa_cq.o` / `efa_rdm_cq.o`, which is the + only reason mocking `efa_ibv_cq_start_poll` / `next_poll` / `end_poll` / + `wc_read_qp_num` / `wc_is_unsolicited` works at all. Co-locate those helpers with + the functions they call and every one of those expectations silently stops + firing. Check with `objdump -r .o | grep `: a relocation means + the seam is live. - **Any efa provider function (not just libibverbs/efadv) can be wrapped — so choose the seam deliberately.** Arming means an unarmed wrapped symbol stays real, but a seam close to the unit under test still keeps error injection @@ -119,11 +129,47 @@ contract or copy a pattern from a nearby test when the docs can settle it. `efa_test_resource_destruct`, or `self_ah`'s real destroy routes into the mock as an unexpected call (see `EfaConnTest`). - **Static-inline functions** (`efa_qp_post_*`, `efa_ibv_cq_*` in - `efa_data_path_ops.h`) are only linkable under `#if EFA_UNIT_TEST`, which turns - their `static inline` bodies into extern decls backed by the stub - `efa_unit_test_data_path_ops.c`. `EFA_UNIT_TEST` is derived from *either* test - suite (`--enable-efa-gtest` OR `--enable-efa-unit-test`), so `--enable-efa-gtest` - alone makes them `--wrap`-able — the gtest suite does **not** need cmocka. + `efa_data_path_ops.h`) are only linkable under `#if EFA_UNIT_TEST`, which gives + them external linkage instead of `static inline` — `EFA_PROD_STATIC_INLINE` + selects which. Exactly one translation unit may then define them, so their + bodies sit behind `EFA_DATA_PATH_OPS_EMIT_BODIES`, defined only by + `efa_unit_test_data_path_ops.c` and necessarily before any include that could + reach the header. That file emits the definitions and calls none of them, which + is what leaves every caller with the undefined reference `--wrap` needs. + `EFA_UNIT_TEST` is derived from *either* test suite (`--enable-efa-gtest` OR + `--enable-efa-unit-test`), so `--enable-efa-gtest` alone makes them + `--wrap`-able — the gtest suite does **not** need cmocka. +- **An unmocked data path op reaches the device.** These wrappers are the same + text the production build compiles; there are no stubs behind them. A fixture + that is not about the data path calls `efa_test_arm_inert_data_path(mock)` right + after `MockEfa::set`, which arms every one of them with a mock that does + nothing. Those expectations are `WillRepeatedly`, so they neither require nor + forbid a call, and gmock matches newest first, so a per-test `EFA_EXPECT_CALL` + added later still wins. A fixture that wants the device just does not call it. + Mocking only part of the CQ is the dangerous case: mock `start_poll` to succeed + and leave an accessor unarmed, and the accessor reads a completion queue that + was never really polled. That is a segfault, not a wrong value. +- **To observe a device write, redirect the sink and restore it.** Neither + `efa_data_path_direct_post_*` nor `ibv_wr_complete` can be `--wrap`ped: both are + static inline, so there is no symbol to rewrite. `efa_test_dp_probe_*` redirects + whichever sink the selected backend uses — the doorbell register and SQ + descriptor buffer, or the `ibv_qp_ex` work-request vtable — and reports + `submitted()` / `pending()` the same way for both, so a test asserting the + contract does not care which backend it is on. Restore has to undo more than it + redirected: a FI_MORE test deliberately leaves work unsubmitted, so + `sq->num_wqe_pending` and `base_ep->is_wr_started` must be cleared too, or a + later real post rings the real doorbell for a producer counter the device never + saw, or skips `ibv_wr_start` on a session that no longer exists. +- **Capability gates must not depend on test order.** `efa_test_device_supports_rma` + reads the selected device list, which the provider only populates on the first + `fi_getinfo`, so it answers false in a process where no endpoint exists yet. + Probe with a throwaway `fi_getinfo` first (see `probe_selected_device` in + `efa_gtest_fi_more.cc`) or a filter selecting only RMA tests silently skips + them all. Gate on the narrowest capability a test really needs, and gate in the + test rather than in `SetUp` — c5n and g4dn have no RDMA read, c7gn and hpc7g no + RDMA write, so a fixture-wide RMA gate discards the send coverage on all four. + `fi_mr_reg` is subject to the same split: asking for `FI_READ` or `FI_WRITE` + without the matching device capability fails with `-FI_EOPNOTSUPP`. - **Inject OOM with `efa_test_fail_mallocs(ordinals)`**, not a `MockEfa` row. It arms 0-based `malloc` ordinals to return NULL (`{0}` = next malloc, `{1,3}` = 2nd and 4th; each failure is one-shot); all others hit `__real_malloc`. Counting diff --git a/prov/efa/test/gtest/efa_gtest_common_helpers.c b/prov/efa/test/gtest/efa_gtest_common_helpers.c index 437f8af296e..cb085d313b6 100644 --- a/prov/efa/test/gtest/efa_gtest_common_helpers.c +++ b/prov/efa/test/gtest/efa_gtest_common_helpers.c @@ -132,6 +132,22 @@ int efa_test_device_supports_rma(void) efa_device_support_rdma_write(); } +int efa_test_device_supports_rdma_read(void) +{ + if (g_efa_selected_device_cnt <= 0) + return 0; + + return efa_device_support_rdma_read(); +} + +int efa_test_device_supports_rdma_write(void) +{ + if (g_efa_selected_device_cnt <= 0) + return 0; + + return efa_device_support_rdma_write(); +} + size_t efa_test_ope_list_count(struct fid_ep *ep) { struct efa_base_ep *base_ep = diff --git a/prov/efa/test/gtest/efa_gtest_common_helpers.h b/prov/efa/test/gtest/efa_gtest_common_helpers.h index 6d37fc23f48..b22c678e8c4 100644 --- a/prov/efa/test/gtest/efa_gtest_common_helpers.h +++ b/prov/efa/test/gtest/efa_gtest_common_helpers.h @@ -37,6 +37,16 @@ void efa_test_fabricate_addr(struct fid_ep *ep, struct efa_ep_addr *addr); */ int efa_test_device_supports_rma(void); +/** + * @brief Whether the real selected EFA device advertises RDMA read. + */ +int efa_test_device_supports_rdma_read(void); + +/** + * @brief Whether the real selected EFA device advertises RDMA write. + */ +int efa_test_device_supports_rdma_write(void); + /** * @brief Set efa_env.track_mr and return its previous value. */ diff --git a/prov/efa/test/gtest/efa_gtest_common_mocks.cc b/prov/efa/test/gtest/efa_gtest_common_mocks.cc index e3bf33656b3..ee992a75ea3 100644 --- a/prov/efa/test/gtest/efa_gtest_common_mocks.cc +++ b/prov/efa/test/gtest/efa_gtest_common_mocks.cc @@ -2,6 +2,7 @@ /* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */ #include "efa_gtest_common_mocks.h" +#include #include static MockEfa *g_mock_efa = nullptr; @@ -64,3 +65,26 @@ void efa_test_fail_mallocs(const std::vector &ordinals) g_malloc_fail[n] = true; } } + +void efa_test_arm_inert_data_path(MockEfa &mock) +{ + using testing::_; + using testing::Return; + + EFA_EXPECT_CALL(mock, efa_qp_post_recv).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_qp_post_send).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_qp_post_read).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_qp_post_write).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_start_poll).WillRepeatedly(Return(ENOENT)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_next_poll).WillRepeatedly(Return(ENOENT)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_end_poll).WillRepeatedly(Return()); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_opcode).WillRepeatedly(Return(IBV_WC_SEND)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_qp_num).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_vendor_err).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_src_qp).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_slid).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_byte_len).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_wc_flags).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_read_imm_data).WillRepeatedly(Return(0)); + EFA_EXPECT_CALL(mock, efa_ibv_cq_wc_is_unsolicited).WillRepeatedly(Return(false)); +} diff --git a/prov/efa/test/gtest/efa_gtest_common_mocks.h b/prov/efa/test/gtest/efa_gtest_common_mocks.h index 6f8f9f6904a..e8871185abb 100644 --- a/prov/efa/test/gtest/efa_gtest_common_mocks.h +++ b/prov/efa/test/gtest/efa_gtest_common_mocks.h @@ -70,6 +70,8 @@ struct dlist_entry; (ibv_cq)) \ X(uint32_t, efa_ibv_cq_wc_read_slid, (struct efa_ibv_cq * ibv_cq), \ (ibv_cq)) \ + X(bool, efa_ibv_cq_wc_is_unsolicited, (struct efa_ibv_cq * ibv_cq), \ + (ibv_cq)) \ X(int, ofi_mr_map_insert, \ (struct ofi_mr_map * map, const struct fi_mr_attr *attr, \ uint64_t *key, void *context, uint64_t flags), \ @@ -173,4 +175,15 @@ EFA_MOCK_FUNCTIONS(EFA_MOCK_GEN_REAL_DECL) */ void efa_test_fail_mallocs(const std::vector &ordinals); +/** + * @brief Arm every data path op with a mock that does nothing. + * + * The data path ops reach the real device when unmocked, so a fixture that is + * not about the data path calls this to keep the device out of it: posts report + * success without submitting, and the CQ reports no completion. Expectations are + * WillRepeatedly, so they neither require nor forbid any call, and a later + * EFA_EXPECT_CALL on the same op takes precedence over them. + */ +void efa_test_arm_inert_data_path(MockEfa &mock); + #endif /* EFA_GTEST_COMMON_MOCKS_H */ diff --git a/prov/efa/test/gtest/efa_gtest_cq.cc b/prov/efa/test/gtest/efa_gtest_cq.cc index 45b35868c60..eae31b8b00b 100644 --- a/prov/efa/test/gtest/efa_gtest_cq.cc +++ b/prov/efa/test/gtest/efa_gtest_cq.cc @@ -34,6 +34,7 @@ class EfaCqTest : public Test qp_num = efa_test_get_qp_num(resource.ep); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override @@ -163,6 +164,7 @@ class EfaCQPollTest : public Test qp_num = efa_test_get_qp_num(resource.ep); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override diff --git a/prov/efa/test/gtest/efa_gtest_fi_more.cc b/prov/efa/test/gtest/efa_gtest_fi_more.cc new file mode 100644 index 00000000000..4223af233b8 --- /dev/null +++ b/prov/efa/test/gtest/efa_gtest_fi_more.cc @@ -0,0 +1,544 @@ +/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */ +/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All + * rights reserved. */ + +/* + * FI_MORE on the transmit path: a work request must be staged without notifying + * the device, and a later post without FI_MORE must submit the whole batch. + * + * Both arms of the efa_qp_post_* dispatch implement that same contract, so the + * tests are written once and parameterized over the backend. Only where the + * submission is observed differs: the direct data path rings the send queue + * doorbell, the rdma-core path calls ibv_wr_complete, and efa_test_dp_probe + * hides which of the two it is watching. + * + * These are the straight-through cases. EfaRdmOpeQueuedFiMoreTest in + * efa_gtest_rdm_ope.cc covers the different claim that an op queued in software + * must not carry a stale FI_MORE when it is later reposted. + */ + +#include "efa_gtest_common_helpers.h" +#include "efa_gtest_common_resource.h" +#include "efa_gtest_fi_more_helpers.h" +#include +#include +#include +#include +#include +#include + +using testing::Combine; +using testing::TestWithParam; +using testing::Values; + +static constexpr size_t kBufSize = 4096; + +/** + * @brief The efa_test_device_supports_* helpers read the selected device list, + * which the provider only populates on the first fi_getinfo. Probe with a + * throwaway one so an answer does not depend on whether an earlier test + * happened to run. + */ +static void probe_selected_device() +{ + struct fi_info *hints = efa_test_alloc_default_hints( + FI_EP_RDM, EFA_DIRECT_FABRIC_NAME); + struct fi_info *info = nullptr; + + if (hints) { + fi_getinfo(FI_VERSION(2, 0), nullptr, nullptr, 0, hints, &info); + fi_freeinfo(info); + fi_freeinfo(hints); + } +} + +static bool device_supports_rdma_read() +{ + if (!efa_test_device_supports_rdma_read()) + probe_selected_device(); + return efa_test_device_supports_rdma_read(); +} + +static bool device_supports_rdma_write() +{ + if (!efa_test_device_supports_rdma_write()) + probe_selected_device(); + return efa_test_device_supports_rdma_write(); +} + +/* efa-direct advertises FI_RMA only when the device has both. */ +static bool device_supports_rma() +{ + return device_supports_rdma_read() && device_supports_rdma_write(); +} + +using EfaFiMoreParam = std::tuple; + +static std::string efa_test_param_name( + const testing::TestParamInfo &info) +{ + std::string name; + + switch (std::get<0>(info.param)) { + case EFA_TEST_POST_SEND: + name = "send"; + break; + case EFA_TEST_POST_READ: + name = "read"; + break; + default: + name = "write"; + break; + } + return name + (std::get<1>(info.param) == EFA_TEST_DP_DIRECT ? + "_direct" : "_rdma_core"); +} + +/** + * @brief An efa-direct endpoint whose selected backend has had its device sink + * redirected, so a post can be observed without reaching the hardware. + * + * efa-direct passes the caller's flags through to the QP post unchanged, which + * is what makes it the right fabric for testing the submission decision itself. + */ +class EfaFiMoreTest : public TestWithParam +{ + protected: + struct efa_resource resource = {}; + struct efa_test_dp_probe probe = {}; + fi_addr_t peer_addr = FI_ADDR_NOTAVAIL; + uint8_t *buf = nullptr; + struct fid_mr *mr = nullptr; + void *desc = nullptr; + int prev_track_mr = 0; + + int op() const { return std::get<0>(GetParam()); } + int backend() const { return std::get<1>(GetParam()); } + + void SetUp() override + { + if (op() != EFA_TEST_POST_SEND && !device_supports_rma()) + GTEST_SKIP() << "device does not support RDMA read+write"; + + memset(&resource, 0, sizeof(resource)); + /* Without MR tracking a post allocates no direct ope, so nothing + * is left outstanding for a completion nothing here reaps. */ + prev_track_mr = efa_test_set_track_mr(0); + + uint64_t mr_access = FI_SEND | FI_RECV; + struct fi_info *hints = efa_test_alloc_default_hints( + FI_EP_RDM, EFA_DIRECT_FABRIC_NAME); + ASSERT_NE(hints, nullptr); + if (op() != EFA_TEST_POST_SEND) { + /* FI_RX_CQ_DATA is required alongside FI_RMA when the + * device lacks unsolicited-write-recv support. */ + hints->caps |= FI_RMA; + hints->mode |= FI_RX_CQ_DATA; + /* A device without RDMA support rejects a registration + * asking for RMA access outright, and the send case has + * no use for it. */ + mr_access |= FI_READ | FI_WRITE; + } + + ASSERT_NO_FATAL_FAILURE( + efa_test_resource_construct(&resource, hints)); + ASSERT_NE(resource.ep, nullptr); + + ASSERT_EQ(efa_test_av_insert_self(resource.ep, resource.av, + &peer_addr), + 1); + + buf = (uint8_t *) calloc(kBufSize, 1); + ASSERT_NE(buf, nullptr); + int ret = fi_mr_reg(resource.domain, buf, kBufSize, mr_access, + 0, 0, 0, &mr, NULL); + ASSERT_EQ(ret, 0) << "fi_mr_reg failed: " << fi_strerror(-ret); + desc = fi_mr_desc(mr); + + ret = efa_test_dp_probe_install(resource.ep, backend(), &probe); + if (ret == -FI_EOPNOTSUPP) + GTEST_SKIP() << "backend not available on this device"; + ASSERT_EQ(ret, 0); + } + + void TearDown() override + { + efa_test_dp_probe_restore(&probe); + if (mr) { + EXPECT_EQ(fi_close(&mr->fid), 0); + mr = nullptr; + } + free(buf); + buf = nullptr; + efa_test_resource_destruct(&resource); + efa_test_set_track_mr(prev_track_mr); + } + + bool submitted() const { return efa_test_dp_probe_submitted(&probe); } + bool pending() const { return efa_test_dp_probe_pending(&probe); } + + /* fi_sendmsg / fi_readmsg / fi_writemsg: the only variants that can + * carry FI_MORE, since it is a per-call flag. */ + ssize_t post_msg(uint64_t flags) + { + struct iovec iov = {.iov_base = buf, .iov_len = kBufSize}; + + if (op() == EFA_TEST_POST_SEND) { + struct fi_msg msg = {}; + + msg.msg_iov = &iov; + msg.desc = &desc; + msg.iov_count = 1; + msg.addr = peer_addr; + return fi_sendmsg(resource.ep, &msg, flags); + } + + struct fi_rma_iov rma_iov = {}; + struct fi_msg_rma msg = {}; + + rma_iov.addr = EFA_TEST_RADDR; + rma_iov.len = kBufSize; + rma_iov.key = EFA_TEST_RKEY; + msg.msg_iov = &iov; + msg.desc = &desc; + msg.iov_count = 1; + msg.addr = peer_addr; + msg.rma_iov = &rma_iov; + msg.rma_iov_count = 1; + + if (op() == EFA_TEST_POST_READ) + return fi_readmsg(resource.ep, &msg, flags); + return fi_writemsg(resource.ep, &msg, flags); + } + + /* The variants with no flags argument. FI_MORE is not among EFA's + * advertised tx op_flags, so these can never defer. */ + ssize_t post_plain() + { + switch (op()) { + case EFA_TEST_POST_SEND: + return fi_send(resource.ep, buf, kBufSize, desc, + peer_addr, NULL); + case EFA_TEST_POST_READ: + return fi_read(resource.ep, buf, kBufSize, desc, + peer_addr, EFA_TEST_RADDR, EFA_TEST_RKEY, + NULL); + default: + return fi_write(resource.ep, buf, kBufSize, desc, + peer_addr, EFA_TEST_RADDR, + EFA_TEST_RKEY, NULL); + } + } +}; + +TEST_P(EfaFiMoreTest, fi_more_defers_submission) +{ + EXPECT_EQ(post_msg(FI_MORE), 0); + + EXPECT_TRUE(pending()); + EXPECT_FALSE(submitted()); +} + +TEST_P(EfaFiMoreTest, no_fi_more_submits) +{ + EXPECT_EQ(post_msg(0), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +TEST_P(EfaFiMoreTest, fi_more_batch_submits_once) +{ + /* Nothing may be submitted while the batch is still being built. A + * submission at any point here would show up as either a rung doorbell + * or a completed work request session. */ + for (int i = 0; i < 3; i++) { + ASSERT_EQ(post_msg(FI_MORE), 0) << "post " << i; + ASSERT_TRUE(pending()) << "post " << i; + ASSERT_FALSE(submitted()) << "post " << i; + } + + EXPECT_EQ(post_msg(0), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +TEST_P(EfaFiMoreTest, plain_call_submits) +{ + EXPECT_EQ(post_plain(), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +/** + * @brief A failed submission still ends the batch, so the work request session + * must not be left open. Only the rdma-core path can fail this way; the direct + * path writes a doorbell register, which reports nothing. + */ +TEST_P(EfaFiMoreTest, submit_failure_clears_wr_session) +{ + if (backend() != EFA_TEST_DP_RDMA_CORE) + GTEST_SKIP() << "submission cannot fail on the direct path"; + + efa_test_dp_probe_set_submit_error(&probe, EINVAL); + EXPECT_NE(post_msg(0), 0); + EXPECT_FALSE(pending()); + + /* So the next post opens a new session rather than reusing a dead one. */ + efa_test_dp_probe_set_submit_error(&probe, 0); + EXPECT_EQ(post_msg(FI_MORE), 0); + EXPECT_TRUE(pending()); +} + +INSTANTIATE_TEST_SUITE_P(Ops, EfaFiMoreTest, + Combine(Values(EFA_TEST_POST_SEND, EFA_TEST_POST_READ, + EFA_TEST_POST_WRITE), + Values(EFA_TEST_DP_DIRECT, + EFA_TEST_DP_RDMA_CORE)), + efa_test_param_name); + +/* ------------------------------------------------------------------------- + * efa-rdm: which operations FI_MORE survives + * ---------------------------------------------------------------------- */ + +/** + * @brief efa-rdm does not forward FI_MORE unconditionally, and the consequence + * of dropping it is observable in the same place as everything above: whether + * the operation was handed to the device or is still waiting. + * + * This is also the only fabric offering tagged operations, so fi_tsendmsg has to + * be tested here. + * + * Only the RMA cases need a device that can do RMA, so they gate themselves + * rather than the fixture: the send cases are worth running on the hardware + * that cannot. + */ +class EfaFiMoreRdmTest : public TestWithParam +{ + protected: + struct efa_resource resource = {}; + struct efa_test_dp_probe probe = {}; + fi_addr_t peer_addr = FI_ADDR_NOTAVAIL; + uint8_t *buf = nullptr; + struct fid_mr *mr = nullptr; + void *desc = nullptr; + + /* Small enough to be an eager RTM, which is the only packet type + * efa-rdm honors FI_MORE for. */ + static constexpr size_t kEagerLen = 32; + /* + * Above any plausible MTU so the message needs a medium RTM, and below + * max_medium_msg_size (64 KiB) so it does not become longcts. + */ + static constexpr size_t kMediumLen = 32768; + static constexpr size_t kRdmBufSize = kMediumLen; + + int backend() const { return GetParam(); } + + void SetUp() override + { + memset(&resource, 0, sizeof(resource)); + + struct fi_info *hints = + efa_test_alloc_default_hints(FI_EP_RDM, EFA_FABRIC_NAME); + ASSERT_NE(hints, nullptr); + /* Asking for FI_TAGGED also leaves peer_may_have_zcpy_rx false, + * so a send does not stall waiting for a handshake. */ + hints->caps |= FI_MSG | FI_TAGGED | FI_RMA; + + ASSERT_NO_FATAL_FAILURE(efa_test_resource_construct_no_enable( + &resource, hints)); + ASSERT_NE(resource.ep, nullptr); + + /* Keep traffic on the device rather than shm. */ + bool shm_permitted = false; + ASSERT_EQ(fi_setopt(&resource.ep->fid, FI_OPT_ENDPOINT, + FI_OPT_SHARED_MEMORY_PERMITTED, + &shm_permitted, sizeof(shm_permitted)), + 0); + ASSERT_EQ(fi_enable(resource.ep), 0); + + ASSERT_EQ(efa_test_rdm_setup_peer(resource.ep, resource.av, + &peer_addr), + 0); + + buf = (uint8_t *) calloc(kRdmBufSize, 1); + ASSERT_NE(buf, nullptr); + /* Asking for access the device cannot provide fails the + * registration outright, so ask for what it has. The tests + * needing more than that skip themselves. */ + uint64_t mr_access = FI_SEND | FI_RECV; + if (device_supports_rdma_read()) + mr_access |= FI_READ; + if (device_supports_rdma_write()) + mr_access |= FI_WRITE; + int ret = fi_mr_reg(resource.domain, buf, kRdmBufSize, mr_access, + 0, 0, 0, &mr, NULL); + ASSERT_EQ(ret, 0) << "fi_mr_reg failed: " << fi_strerror(-ret); + desc = fi_mr_desc(mr); + + ret = efa_test_dp_probe_install(resource.ep, backend(), &probe); + if (ret == -FI_EOPNOTSUPP) + GTEST_SKIP() << "backend not available on this device"; + ASSERT_EQ(ret, 0); + } + + void TearDown() override + { + efa_test_dp_probe_restore(&probe); + if (mr) { + EXPECT_EQ(fi_close(&mr->fid), 0); + mr = nullptr; + } + free(buf); + buf = nullptr; + efa_test_resource_destruct(&resource); + } + + bool submitted() const { return efa_test_dp_probe_submitted(&probe); } + bool pending() const { return efa_test_dp_probe_pending(&probe); } + + ssize_t tsendmsg(size_t len, uint64_t flags) + { + struct iovec iov = {.iov_base = buf, .iov_len = len}; + struct fi_msg_tagged tmsg = {}; + + tmsg.msg_iov = &iov; + tmsg.desc = &desc; + tmsg.iov_count = 1; + tmsg.addr = peer_addr; + tmsg.tag = 0x1234; + return fi_tsendmsg(resource.ep, &tmsg, flags); + } + + ssize_t sendmsg(size_t len, uint64_t flags) + { + struct iovec iov = {.iov_base = buf, .iov_len = len}; + struct fi_msg msg = {}; + + msg.msg_iov = &iov; + msg.desc = &desc; + msg.iov_count = 1; + msg.addr = peer_addr; + return fi_sendmsg(resource.ep, &msg, flags); + } + + ssize_t writemsg(size_t len, uint64_t flags) + { + return rma(len, flags, /* is_read */ false); + } + + ssize_t readmsg(size_t len, uint64_t flags) + { + return rma(len, flags, /* is_read */ true); + } + + ssize_t rma(size_t len, uint64_t flags, bool is_read) + { + struct iovec iov = {.iov_base = buf, .iov_len = len}; + struct fi_rma_iov rma_iov = {}; + struct fi_msg_rma msg = {}; + + rma_iov.addr = EFA_TEST_RADDR; + rma_iov.len = len; + rma_iov.key = EFA_TEST_RKEY; + msg.msg_iov = &iov; + msg.desc = &desc; + msg.iov_count = 1; + msg.addr = peer_addr; + msg.rma_iov = &rma_iov; + msg.rma_iov_count = 1; + return is_read ? fi_readmsg(resource.ep, &msg, flags) : + fi_writemsg(resource.ep, &msg, flags); + } +}; + +TEST_P(EfaFiMoreRdmTest, eager_tsendmsg_fi_more_defers_submission) +{ + EXPECT_EQ(tsendmsg(kEagerLen, FI_MORE), 0); + + EXPECT_TRUE(pending()); + EXPECT_FALSE(submitted()); +} + +TEST_P(EfaFiMoreRdmTest, eager_tsendmsg_without_fi_more_submits) +{ + EXPECT_EQ(tsendmsg(kEagerLen, 0), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +/** + * @brief A read the application issued honors FI_MORE like a write does. + * + * A read posted for an rxe does not, because that belongs to a receive-side + * protocol such as longread: its flags describe the fi_recv, not a stream of + * transmits being batched, so deferring it would wait for a flush that never + * comes. Only the application-issued case is reachable from here. + */ +TEST_P(EfaFiMoreRdmTest, readmsg_fi_more_defers_submission) +{ + if (!device_supports_rdma_read()) + GTEST_SKIP() << "device does not support RDMA read"; + + EXPECT_EQ(readmsg(kEagerLen, FI_MORE), 0); + + EXPECT_TRUE(pending()); + EXPECT_FALSE(submitted()); +} + +TEST_P(EfaFiMoreRdmTest, readmsg_without_fi_more_submits) +{ + if (!device_supports_rdma_read()) + GTEST_SKIP() << "device does not support RDMA read"; + + EXPECT_EQ(readmsg(kEagerLen, 0), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +TEST_P(EfaFiMoreRdmTest, writemsg_fi_more_defers_submission) +{ + if (!device_supports_rdma_write()) + GTEST_SKIP() << "device does not support RDMA write"; + + EXPECT_EQ(writemsg(kEagerLen, FI_MORE), 0); + + EXPECT_TRUE(pending()); + EXPECT_FALSE(submitted()); +} + +TEST_P(EfaFiMoreRdmTest, writemsg_without_fi_more_submits) +{ + if (!device_supports_rdma_write()) + GTEST_SKIP() << "device does not support RDMA write"; + + EXPECT_EQ(writemsg(kEagerLen, 0), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +/** + * @brief FI_MORE is forwarded only for eager packet types. Honoring it for a + * medium RTM would leave the last packet of the message carrying it, so nothing + * would flush the message at all. + */ +TEST_P(EfaFiMoreRdmTest, non_eager_sendmsg_submits_despite_fi_more) +{ + EXPECT_EQ(sendmsg(kMediumLen, FI_MORE), 0); + + EXPECT_FALSE(pending()); + EXPECT_TRUE(submitted()); +} + +INSTANTIATE_TEST_SUITE_P(Backends, EfaFiMoreRdmTest, + Values(EFA_TEST_DP_DIRECT, EFA_TEST_DP_RDMA_CORE), + [](const testing::TestParamInfo &info) { + return info.param == EFA_TEST_DP_DIRECT ? + "direct" : "rdma_core"; + }); diff --git a/prov/efa/test/gtest/efa_gtest_fi_more_helpers.c b/prov/efa/test/gtest/efa_gtest_fi_more_helpers.c new file mode 100644 index 00000000000..df928209362 --- /dev/null +++ b/prov/efa/test/gtest/efa_gtest_fi_more_helpers.c @@ -0,0 +1,365 @@ +/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */ +/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */ + +#include "efa_gtest_fi_more_helpers.h" +#include "efa.h" +#include "efa_ah.h" +#include "efa_av.h" +#include "efa_mr.h" +#include "efa_base_ep.h" +#include "efa_cq.h" +#include "efa_data_path_ops.h" +#include "efa_io_defs.h" +#include "rdm/efa_rdm_ep.h" +#include "rdm/efa_rdm_peer.h" +#include "rdm/efa_rdm_protocol.h" + +/* Seeded into the redirected doorbell, so "not rung" is distinguishable. */ +#define EFA_TEST_SQ_DB_SENTINEL 0xDEADBEEFu + +static struct efa_base_ep *efa_test_base_ep_from_ep(struct fid_ep *ep) +{ + return container_of(ep, struct efa_base_ep, util_ep.ep_fid); +} + +int efa_test_ep_is_wr_started(struct fid_ep *ep) +{ + return efa_test_base_ep_from_ep(ep)->is_wr_started; +} + +/* --------------------------------------------------------------------------- + * EFA_TEST_DP_RDMA_CORE: the work request vtable + * + * struct ibv_qp_ex carries no user cookie, so the mocked slots have no way to + * reach per-probe state. gtest runs one test at a time in one thread, and + * install() resets these, so one set of file-static counters is enough. + * ------------------------------------------------------------------------ */ + +static unsigned g_wr_complete_cnt; +static int g_wr_complete_err; + +static void efa_test_mock_wr_start(struct ibv_qp_ex *qpx) +{ +} + +static int efa_test_mock_wr_complete(struct ibv_qp_ex *qpx) +{ + g_wr_complete_cnt++; + return g_wr_complete_err; +} + +static void efa_test_mock_wr_send(struct ibv_qp_ex *qpx) +{ +} + +static void efa_test_mock_wr_send_imm(struct ibv_qp_ex *qpx, __be32 imm_data) +{ +} + +static void efa_test_mock_wr_rdma_read(struct ibv_qp_ex *qpx, uint32_t rkey, + uint64_t remote_addr) +{ +} + +static void efa_test_mock_wr_rdma_write(struct ibv_qp_ex *qpx, uint32_t rkey, + uint64_t remote_addr) +{ +} + +static void efa_test_mock_wr_rdma_write_imm(struct ibv_qp_ex *qpx, + uint32_t rkey, + uint64_t remote_addr, + __be32 imm_data) +{ +} + +static void efa_test_mock_wr_set_sge_list(struct ibv_qp_ex *qpx, size_t num_sge, + const struct ibv_sge *sg_list) +{ +} + +static void efa_test_mock_wr_set_inline_data_list( + struct ibv_qp_ex *qpx, size_t num_buf, + const struct ibv_data_buf *buf_list) +{ +} + +static void efa_test_mock_wr_set_ud_addr(struct ibv_qp_ex *qpx, + struct ibv_ah *ah, + uint32_t remote_qpn, + uint32_t remote_qkey) +{ +} + +#if HAVE_EFADV_WR_PROCESSING_HINTS +static void efa_test_mock_wr_set_processing_hints(struct efadv_qp *efadv_qp, + uint32_t hints) +{ +} +#endif + +static int efa_test_wr_probe_install(struct efa_qp *qp, + struct efa_test_dp_probe *p) +{ + struct ibv_qp_ex *qpx; + + if (!qp->ibv_qp_ex) + return -FI_EOPNOTSUPP; + + qpx = qp->ibv_qp_ex; + p->saved_qpx = aligned_alloc(_Alignof(struct ibv_qp_ex), + sizeof(struct ibv_qp_ex)); + if (!p->saved_qpx) + return -FI_ENOMEM; + memcpy(p->saved_qpx, qpx, sizeof(*qpx)); + p->qpx = qpx; + +#if HAVE_EFA_DATA_PATH_DIRECT + p->saved_direct_enabled = qp->data_path_direct_enabled; + /* Select the rdma-core arm of the efa_qp_post_* dispatch. */ + qp->data_path_direct_enabled = false; +#endif + + g_wr_complete_cnt = 0; + g_wr_complete_err = 0; + + qpx->wr_start = &efa_test_mock_wr_start; + qpx->wr_complete = &efa_test_mock_wr_complete; + qpx->wr_send = &efa_test_mock_wr_send; + qpx->wr_send_imm = &efa_test_mock_wr_send_imm; + qpx->wr_rdma_read = &efa_test_mock_wr_rdma_read; + qpx->wr_rdma_write = &efa_test_mock_wr_rdma_write; + qpx->wr_rdma_write_imm = &efa_test_mock_wr_rdma_write_imm; + qpx->wr_set_sge_list = &efa_test_mock_wr_set_sge_list; + qpx->wr_set_inline_data_list = &efa_test_mock_wr_set_inline_data_list; + qpx->wr_set_ud_addr = &efa_test_mock_wr_set_ud_addr; + +#if HAVE_EFADV_WR_PROCESSING_HINTS + { + struct efadv_qp *efadv_qp = efadv_qp_from_ibv_qp_ex(qpx); + + p->saved_set_hints = efadv_qp->wr_set_processing_hints; + efadv_qp->wr_set_processing_hints = + &efa_test_mock_wr_set_processing_hints; + } +#endif + return 0; +} + +static void efa_test_wr_probe_restore(struct efa_test_dp_probe *p) +{ + if (!p->qpx) + return; + +#if HAVE_EFADV_WR_PROCESSING_HINTS + efadv_qp_from_ibv_qp_ex(p->qpx)->wr_set_processing_hints = + p->saved_set_hints; +#endif + memcpy(p->qpx, p->saved_qpx, sizeof(struct ibv_qp_ex)); + free(p->saved_qpx); + p->saved_qpx = NULL; + p->qpx = NULL; + +#if HAVE_EFA_DATA_PATH_DIRECT + ((struct efa_qp *) p->qp)->data_path_direct_enabled = + p->saved_direct_enabled; +#endif +} + +/* --------------------------------------------------------------------------- + * EFA_TEST_DP_DIRECT: the send queue descriptor buffer and doorbell + * ------------------------------------------------------------------------ */ + +#if HAVE_EFA_DATA_PATH_DIRECT + +static int efa_test_sq_probe_install(struct efa_qp *qp, + struct efa_test_dp_probe *p) +{ + struct efa_data_path_direct_sq *sq; + size_t bytes; + + if (!qp->data_path_direct_enabled) + return -FI_EOPNOTSUPP; + + sq = &qp->data_path_direct_qp.sq; + bytes = (size_t) sq->wq.wqe_cnt * sq->wq.wqe_size; + + /* + * aligned_alloc, not malloc: __wrap_malloc counts ordinals for + * efa_test_fail_mallocs, and the probe must not shift that count. + * 64-byte alignment satisfies mmio_memcpy_x64, which is vst4q_u64 on + * aarch64. + */ + p->scratch_desc = aligned_alloc(64, bytes); + if (!p->scratch_desc) + return -FI_ENOMEM; + memset(p->scratch_desc, 0, bytes); + + p->sq = sq; + p->saved_desc = sq->desc; + p->saved_db = sq->wq.db; + p->scratch_db = EFA_TEST_SQ_DB_SENTINEL; + + sq->desc = p->scratch_desc; + sq->wq.db = &p->scratch_db; + return 0; +} + +static void efa_test_sq_probe_restore(struct efa_test_dp_probe *p) +{ + struct efa_data_path_direct_sq *sq = p->sq; + + if (!sq) + return; + + sq->desc = p->saved_desc; + sq->wq.db = p->saved_db; + /* + * A FI_MORE test deliberately leaves entries staged. Clear the count, or + * a later real post takes the pending branch and rings the real doorbell + * for a producer counter the device never saw. + */ + sq->num_wqe_pending = 0; + + free(p->scratch_desc); + p->scratch_desc = NULL; + p->sq = NULL; +} + +static bool efa_test_sq_probe_pending(const struct efa_test_dp_probe *p) +{ + return ((struct efa_data_path_direct_sq *) p->sq)->num_wqe_pending != 0; +} + +#else /* !HAVE_EFA_DATA_PATH_DIRECT */ + +static int efa_test_sq_probe_install(struct efa_qp *qp, + struct efa_test_dp_probe *p) +{ + return -FI_EOPNOTSUPP; +} + +static void efa_test_sq_probe_restore(struct efa_test_dp_probe *p) +{ +} + +static bool efa_test_sq_probe_pending(const struct efa_test_dp_probe *p) +{ + return false; +} + +#endif /* HAVE_EFA_DATA_PATH_DIRECT */ + +/* --------------------------------------------------------------------------- + * Backend-agnostic probe + * ------------------------------------------------------------------------ */ + +int efa_test_dp_probe_install(struct fid_ep *ep, int backend, + struct efa_test_dp_probe *p) +{ + struct efa_qp *qp = efa_test_base_ep_from_ep(ep)->qp; + int ret; + + memset(p, 0, sizeof(*p)); + p->backend = backend; + + if (!qp) + return -FI_EOPNOTSUPP; + p->qp = qp; + + ret = (backend == EFA_TEST_DP_DIRECT) ? + efa_test_sq_probe_install(qp, p) : + efa_test_wr_probe_install(qp, p); + if (ret) + p->qp = NULL; + return ret; +} + +void efa_test_dp_probe_restore(struct efa_test_dp_probe *p) +{ + struct efa_qp *qp = p->qp; + + if (!qp) + return; + + if (p->backend == EFA_TEST_DP_DIRECT) + efa_test_sq_probe_restore(p); + else + efa_test_wr_probe_restore(p); + + /* + * Closing the endpoint does not flush an open ibv_wr_start session, so a + * later real post would skip ibv_wr_start on a session that no longer + * exists. + */ + qp->base_ep->is_wr_started = false; + p->qp = NULL; +} + +bool efa_test_dp_probe_submitted(const struct efa_test_dp_probe *p) +{ + if (p->backend == EFA_TEST_DP_DIRECT) + return p->scratch_db != EFA_TEST_SQ_DB_SENTINEL; + return g_wr_complete_cnt > 0; +} + +bool efa_test_dp_probe_pending(const struct efa_test_dp_probe *p) +{ + if (p->backend == EFA_TEST_DP_DIRECT) + return efa_test_sq_probe_pending(p); + return ((struct efa_qp *) p->qp)->base_ep->is_wr_started; +} + +void efa_test_dp_probe_reset(struct efa_test_dp_probe *p) +{ + if (p->backend == EFA_TEST_DP_DIRECT) + p->scratch_db = EFA_TEST_SQ_DB_SENTINEL; + else + g_wr_complete_cnt = 0; +} + +void efa_test_dp_probe_set_submit_error(struct efa_test_dp_probe *p, int err) +{ + g_wr_complete_err = err; +} + +/* --------------------------------------------------------------------------- + * efa-rdm + * ------------------------------------------------------------------------ */ + +int efa_test_rdm_setup_peer(struct fid_ep *ep_fid, struct fid_av *av_fid, + fi_addr_t *peer_addr) +{ + struct efa_rdm_ep *ep = container_of(ep_fid, struct efa_rdm_ep, + base_ep.util_ep.ep_fid); + struct efa_ep_addr raw_addr = {0}; + size_t raw_addr_len = sizeof(raw_addr); + struct efa_rdm_peer *peer; + int ret; + + ret = fi_getname(&ep_fid->fid, &raw_addr, &raw_addr_len); + if (ret) + return ret; + /* Own GID with a different QPN: the AH is created against the real + * device, but the peer is not self, so shm is not a candidate. */ + raw_addr.qpn = 1; + raw_addr.qkey = 0x1234; + if (fi_av_insert(av_fid, &raw_addr, 1, peer_addr, 0, NULL) != 1) + return -FI_EINVAL; + + peer = efa_rdm_ep_get_peer_explicit(ep, *peer_addr); + if (!peer) + return -FI_EINVAL; + + peer->flags |= EFA_RDM_PEER_HANDSHAKE_RECEIVED; + peer->extra_info[0] |= EFA_RDM_EXTRA_FEATURE_RDMA_READ | + EFA_RDM_EXTRA_FEATURE_RDMA_WRITE; + peer->p2p_supported = true; + if (peer->conn) + peer->conn->shm_fi_addr = FI_ADDR_NOTAVAIL; + /* use_device_rdma defaults off on some platforms, which would route a + * read or write through the emulated protocols instead of the QP. */ + ep->use_device_rdma = true; + + return 0; +} diff --git a/prov/efa/test/gtest/efa_gtest_fi_more_helpers.h b/prov/efa/test/gtest/efa_gtest_fi_more_helpers.h new file mode 100644 index 00000000000..eb2732561e5 --- /dev/null +++ b/prov/efa/test/gtest/efa_gtest_fi_more_helpers.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */ +/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */ + +#ifndef EFA_GTEST_FI_MORE_HELPERS_H +#define EFA_GTEST_FI_MORE_HELPERS_H + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Remote addressing for the RMA operations. Nothing is deliverable, so these + * only have to be distinctive. */ +#define EFA_TEST_RKEY 0x00C0FFEE +#define EFA_TEST_RADDR 0x0000BEEF00001000ULL + +enum efa_test_post_op { + EFA_TEST_POST_SEND = 0, + EFA_TEST_POST_READ = 1, + EFA_TEST_POST_WRITE = 2, +}; + +/** + * @brief Which arm of the efa_qp_post_* dispatch to exercise. + * + * The two hand work requests to the device by different means, so a test + * observes them in different places, but the FI_MORE contract they implement is + * the same one. + */ +enum efa_test_dp_backend { + /** efa_data_path_direct_post_*: submitting rings the SQ doorbell. */ + EFA_TEST_DP_DIRECT = 0, + /** efa_ibv_post_*: submitting calls ibv_wr_complete. */ + EFA_TEST_DP_RDMA_CORE = 1, +}; + +/** + * @brief Intercepts however the selected backend hands work to the device. + * + * Opaque to the test body; use the accessors below, which read the same way for + * either backend. + */ +struct efa_test_dp_probe { + int backend; + void *qp; /* struct efa_qp *, NULL when not installed */ + + /* EFA_TEST_DP_DIRECT: the redirected send queue */ + void *sq; /* struct efa_data_path_direct_sq * */ + uint8_t *saved_desc; + uint32_t *saved_db; + uint8_t *scratch_desc; + uint32_t scratch_db; + + /* EFA_TEST_DP_RDMA_CORE: the hooked work request vtable */ + void *qpx; /* struct ibv_qp_ex * */ + void *saved_qpx; /* malloc'd copy of the original vtable */ + void *saved_set_hints; + int saved_direct_enabled; +}; + +/** + * @brief Redirect the selected backend's device sink at test-owned state, so a + * post can be observed without anything reaching the hardware. + * + * For EFA_TEST_DP_DIRECT this points the send queue's descriptor buffer and + * doorbell register at test memory. For EFA_TEST_DP_RDMA_CORE it clears + * qp->data_path_direct_enabled, so that arm of the dispatch is the one taken, + * and replaces the ibv_qp_ex work request vtable with counting no-ops. + * + * @return 0, -FI_EOPNOTSUPP if the backend is unavailable on this device, or + * -FI_ENOMEM on allocation failure. + */ +int efa_test_dp_probe_install(struct fid_ep *ep, int backend, + struct efa_test_dp_probe *p); + +/** + * @brief Undo the sink redirection and clear the staged-work state, so the + * endpoint can be closed cleanly. Idempotent. + * + * Must run before the endpoint is closed: a FI_MORE test + * deliberately leaves work unsubmitted, and closing the endpoint neither rings + * the doorbell nor flushes an open work request session. + */ +void efa_test_dp_probe_restore(struct efa_test_dp_probe *p); + +/** + * @brief Whether the backend has handed anything to the device since install() + * or the last reset(): the doorbell rang, or ibv_wr_complete was called. + */ +bool efa_test_dp_probe_submitted(const struct efa_test_dp_probe *p); + +/** + * @brief Whether the backend is holding work it has not submitted: entries + * staged in the send queue, or an open work request session. + */ +bool efa_test_dp_probe_pending(const struct efa_test_dp_probe *p); + +/** @brief Forget any submission already observed. */ +void efa_test_dp_probe_reset(struct efa_test_dp_probe *p); + +/** + * @brief Make submissions fail with @p err, until called again with 0. + * + * EFA_TEST_DP_RDMA_CORE only: the direct path writes a doorbell register rather + * than calling anything that can report an error. + */ +void efa_test_dp_probe_set_submit_error(struct efa_test_dp_probe *p, int err); + +/** @brief Whether the endpoint has an ibv_wr_start session open. */ +int efa_test_ep_is_wr_started(struct fid_ep *ep); + +/* --------------------------------------------------------------------------- + * efa-rdm + * ------------------------------------------------------------------------ */ + +/** + * @brief Set up an RDM peer that takes the device data path: own GID with a + * different QPN, so the address handle is real but the peer is not self, with + * the handshake recorded as received and device RDMA plus p2p advertised. + */ +int efa_test_rdm_setup_peer(struct fid_ep *ep, struct fid_av *av, + fi_addr_t *peer_addr); + +#ifdef __cplusplus +} +#endif + +#endif /* EFA_GTEST_FI_MORE_HELPERS_H */ diff --git a/prov/efa/test/gtest/efa_gtest_msg.cc b/prov/efa/test/gtest/efa_gtest_msg.cc index 9ec09726f40..dde93f1fc38 100644 --- a/prov/efa/test/gtest/efa_gtest_msg.cc +++ b/prov/efa/test/gtest/efa_gtest_msg.cc @@ -46,6 +46,7 @@ class EfaMsgTest : public Test 1); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void SetUp() override diff --git a/prov/efa/test/gtest/efa_gtest_rdm_cntr.cc b/prov/efa/test/gtest/efa_gtest_rdm_cntr.cc index f8f94a72030..cd2bcc20abd 100644 --- a/prov/efa/test/gtest/efa_gtest_rdm_cntr.cc +++ b/prov/efa/test/gtest/efa_gtest_rdm_cntr.cc @@ -25,6 +25,7 @@ class EfaRdmCntrTest : public Test ASSERT_NE(resource.domain, nullptr); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override diff --git a/prov/efa/test/gtest/efa_gtest_rdm_ope.cc b/prov/efa/test/gtest/efa_gtest_rdm_ope.cc index 8853bebbc5c..fdf6e104eff 100644 --- a/prov/efa/test/gtest/efa_gtest_rdm_ope.cc +++ b/prov/efa/test/gtest/efa_gtest_rdm_ope.cc @@ -92,6 +92,7 @@ class EfaRdmOpeQueuedFiMoreTest : public TestWithParam << "device does not support RDMA read+write"; MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override @@ -183,6 +184,7 @@ class EfaRdmOpeProcessQueuedTest : public Test ASSERT_NE(resource.ep, nullptr); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override @@ -311,6 +313,7 @@ class EfaRdmOpeQueuedFlagDispatchTest : public TestWithParam ASSERT_NE(resource.ep, nullptr); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override diff --git a/prov/efa/test/gtest/efa_gtest_rdm_peer.cc b/prov/efa/test/gtest/efa_gtest_rdm_peer.cc index 7092c52c028..e82f6e27acb 100644 --- a/prov/efa/test/gtest/efa_gtest_rdm_peer.cc +++ b/prov/efa/test/gtest/efa_gtest_rdm_peer.cc @@ -31,6 +31,7 @@ class EfaRdmPeerTest : public testing::Test 1); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override diff --git a/prov/efa/test/gtest/efa_gtest_rdm_pke_rtm.cc b/prov/efa/test/gtest/efa_gtest_rdm_pke_rtm.cc index be1ee073e8f..0fc90f2beb5 100644 --- a/prov/efa/test/gtest/efa_gtest_rdm_pke_rtm.cc +++ b/prov/efa/test/gtest/efa_gtest_rdm_pke_rtm.cc @@ -35,6 +35,7 @@ class EfaRtmRecvTest : public TestWithParam 1); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } void TearDown() override diff --git a/prov/efa/test/gtest/efa_gtest_rma.cc b/prov/efa/test/gtest/efa_gtest_rma.cc index 6fbec13247a..49f617163a8 100644 --- a/prov/efa/test/gtest/efa_gtest_rma.cc +++ b/prov/efa/test/gtest/efa_gtest_rma.cc @@ -42,6 +42,7 @@ class EfaRmaTest : public Test 1); MockEfa::set(&mock_efa); + efa_test_arm_inert_data_path(mock_efa); } /* FI_RX_CQ_DATA is required for FI_RMA when the device lacks