Skip to content

Commit

Permalink
libhns: Support reporting WC as software mode
Browse files Browse the repository at this point in the history 
When HW is being or has been reset, it won't generate cqe anymore,
and users can't poll all work completions as expected.

This patch allows userspace driver to compose the expected WCs
instead of HW in this case. All SQs, RQs and SRQs is linked to a list
respectively. When reset state is detected during polling CQ, walk
the lists and compose software-WCs with error status IBV_WC_WR_FLUSH_ERR
according to the polling number specified by users.

Signed-off-by: Chengchang Tang <[email protected]>
Signed-off-by: Junxian Huang <[email protected]>
  • Loading branch information
Chengchang Tang authored and Junxian Huang committed Oct 14, 2024
1 parent 80d452f commit 0092957
Show file tree
Hide file tree
Showing 4 changed files with 293 additions and 5 deletions.
9 changes: 9 additions & 0 deletions providers/hns/hns_roce_u.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,11 @@ struct hns_roce_cq {
unsigned int cqe_size;
struct hns_roce_v2_cqe *cqe;
struct ibv_pd *parent_domain;
struct list_head list_sq;
struct list_head list_rq;
struct list_head list_srq;
struct list_head list_xrc_srq;
struct hns_roce_v2_cqe *sw_cqe;
};

struct hns_roce_idx_que {
Expand Down Expand Up @@ -302,6 +307,7 @@ struct hns_roce_srq {
unsigned int *rdb;
unsigned int cap_flags;
unsigned short counter;
struct list_node xrc_srcq_node;
};

struct hns_roce_wq {
Expand Down Expand Up @@ -362,6 +368,9 @@ struct hns_roce_qp {
void *cur_wqe;
unsigned int rb_sq_head; /* roll back sq head */
struct hns_roce_sge_info sge_info;
struct list_node rcq_node;
struct list_node scq_node;
struct list_node srcq_node;
};

struct hns_roce_av {
Expand Down
196 changes: 191 additions & 5 deletions providers/hns/hns_roce_u_hw_v2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,170 @@ static int hns_roce_poll_one(struct hns_roce_context *ctx,
return hns_roce_flush_cqe(*cur_qp, status);
}

static void hns_roce_fill_swc(struct hns_roce_cq *cq, struct ibv_wc *wc,
uint64_t wr_id, uint32_t qp_num)
{
if (!wc) {
cq->verbs_cq.cq_ex.status = IBV_WC_WR_FLUSH_ERR;
cq->verbs_cq.cq_ex.wr_id = wr_id;
hr_reg_write(cq->sw_cqe, CQE_LCL_QPN, qp_num);
return;
}

wc->wr_id = wr_id;
wc->status = IBV_WC_WR_FLUSH_ERR;
wc->vendor_err = 0;
wc->qp_num = qp_num;
}

static int hns_roce_get_wq_swc(struct hns_roce_cq *cq, struct hns_roce_qp *qp,
struct ibv_wc *wc, bool is_sq)
{
struct hns_roce_wq *wq = is_sq ? &qp->sq : &qp->rq;
unsigned int left_wr;
uint64_t wr_id;

left_wr = wq->head - wq->tail;
if (left_wr == 0) {
if (is_sq)
list_del_init(&qp->scq_node);
else
list_del_init(&qp->rcq_node);

return ENOENT;
}

wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
hns_roce_fill_swc(cq, wc, wr_id, qp->verbs_qp.qp.qp_num);
wq->tail++;
return V2_CQ_OK;
}

static int hns_roce_gen_sq_swc(struct hns_roce_cq *cq, struct ibv_wc *wc)
{
struct hns_roce_qp *next, *qp = NULL;

list_for_each_safe(&cq->list_sq, qp, next, scq_node) {
if (hns_roce_get_wq_swc(cq, qp, wc, true) == ENOENT)
continue;

return V2_CQ_OK;
}

return !wc ? ENOENT : V2_CQ_EMPTY;
}

static int hns_roce_gen_rq_swc(struct hns_roce_cq *cq, struct ibv_wc *wc)
{
struct hns_roce_qp *next, *qp = NULL;

list_for_each_safe(&cq->list_rq, qp, next, rcq_node) {
if (hns_roce_get_wq_swc(cq, qp, wc, false) == ENOENT)
continue;

return V2_CQ_OK;
}

return !wc ? ENOENT : V2_CQ_EMPTY;
}

static int hns_roce_get_srq_swc(struct hns_roce_cq *cq, struct hns_roce_qp *qp,
struct hns_roce_srq *srq, struct ibv_wc *wc)
{
unsigned int left_wr;
uint64_t wr_id;

hns_roce_spin_lock(&srq->hr_lock);
left_wr = srq->idx_que.head - srq->idx_que.tail;
if (left_wr == 0) {
if (qp)
list_del_init(&qp->srcq_node);
else
list_del_init(&srq->xrc_srcq_node);

hns_roce_spin_unlock(&srq->hr_lock);
return ENOENT;
}

wr_id = srq->wrid[srq->idx_que.tail & (srq->wqe_cnt - 1)];
hns_roce_fill_swc(cq, wc, wr_id, srq->srqn);
srq->idx_que.tail++;
hns_roce_spin_unlock(&srq->hr_lock);

return V2_CQ_OK;
}

static int hns_roce_gen_common_srq_swc(struct hns_roce_cq *cq,
struct ibv_wc *wc)
{
struct hns_roce_qp *next, *qp = NULL;
struct hns_roce_srq *srq;

list_for_each_safe(&cq->list_srq, qp, next, srcq_node) {
srq = to_hr_srq(qp->verbs_qp.qp.srq);
if (hns_roce_get_srq_swc(cq, qp, srq, wc) == ENOENT)
continue;

return V2_CQ_OK;
}

return !wc ? ENOENT : V2_CQ_EMPTY;
}

static int hns_roce_gen_xrc_srq_swc(struct hns_roce_cq *cq, struct ibv_wc *wc)
{
struct hns_roce_srq *next, *srq = NULL;

list_for_each_safe(&cq->list_xrc_srq, srq, next, xrc_srcq_node) {
if (hns_roce_get_srq_swc(cq, NULL, srq, wc) == ENOENT)
continue;

return V2_CQ_OK;
}

return !wc ? ENOENT : V2_CQ_EMPTY;
}

static int hns_roce_gen_srq_swc(struct hns_roce_cq *cq, struct ibv_wc *wc)
{
int err;

err = hns_roce_gen_common_srq_swc(cq, wc);
if (err == V2_CQ_OK)
return err;

return hns_roce_gen_xrc_srq_swc(cq, wc);
}

static int hns_roce_poll_one_swc(struct hns_roce_cq *cq, struct ibv_wc *wc)
{
int err;

err = hns_roce_gen_sq_swc(cq, wc);
if (err == V2_CQ_OK)
return err;

err = hns_roce_gen_rq_swc(cq, wc);
if (err == V2_CQ_OK)
return err;

return hns_roce_gen_srq_swc(cq, wc);
}

static int hns_roce_poll_swc(struct hns_roce_cq *cq, int ne, struct ibv_wc *wc)
{
int npolled;
int err;

for (npolled = 0; npolled < ne; npolled++) {
err = hns_roce_poll_one_swc(cq, wc + npolled);
if (err == V2_CQ_EMPTY)
break;
}

return npolled;
}

static bool hns_roce_is_reset(struct hns_roce_context *ctx)
{
struct hns_roce_reset_state *state = ctx->reset_state;
Expand All @@ -746,6 +910,12 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,

hns_roce_spin_lock(&cq->hr_lock);

if (unlikely(hns_roce_is_reset(ctx))) {
npolled = hns_roce_poll_swc(cq, ne, wc);
hns_roce_spin_unlock(&cq->hr_lock);
return npolled;
}

for (npolled = 0; npolled < ne; ++npolled) {
err = hns_roce_poll_one(ctx, &qp, cq, wc + npolled);
if (err != V2_CQ_OK)
Expand Down Expand Up @@ -1621,7 +1791,7 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
return ret;
}

static void hns_roce_lock_cqs(struct ibv_qp *qp)
void hns_roce_lock_cqs(struct ibv_qp *qp)
{
struct hns_roce_cq *send_cq = to_hr_cq(qp->send_cq);
struct hns_roce_cq *recv_cq = to_hr_cq(qp->recv_cq);
Expand All @@ -1643,7 +1813,7 @@ static void hns_roce_lock_cqs(struct ibv_qp *qp)
}
}

static void hns_roce_unlock_cqs(struct ibv_qp *qp)
void hns_roce_unlock_cqs(struct ibv_qp *qp)
{
struct hns_roce_cq *send_cq = to_hr_cq(qp->send_cq);
struct hns_roce_cq *recv_cq = to_hr_cq(qp->recv_cq);
Expand Down Expand Up @@ -1683,13 +1853,18 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp)

hns_roce_lock_cqs(ibqp);

if (ibqp->recv_cq)
if (ibqp->recv_cq) {
__hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), ibqp->qp_num,
ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
list_del(&qp->srcq_node);
list_del(&qp->rcq_node);
}

if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq)
if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq) {
__hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num,
NULL);
list_del(&qp->scq_node);
}

hns_roce_unlock_cqs(ibqp);

Expand Down Expand Up @@ -1854,7 +2029,11 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current,

hns_roce_spin_lock(&cq->hr_lock);

err = hns_roce_poll_one(ctx, &qp, cq, NULL);
if (unlikely(hns_roce_is_reset(ctx)))
err = hns_roce_poll_one_swc(cq, NULL);
else
err = hns_roce_poll_one(ctx, &qp, cq, NULL);

if (err != V2_CQ_OK)
hns_roce_spin_unlock(&cq->hr_lock);

Expand All @@ -1868,6 +2047,9 @@ static int wc_next_poll_cq(struct ibv_cq_ex *current)
struct hns_roce_qp *qp = NULL;
int err;

if (unlikely(hns_roce_is_reset(ctx)))
return hns_roce_poll_one_swc(cq, NULL);

err = hns_roce_poll_one(ctx, &qp, cq, NULL);
if (err != V2_CQ_OK)
return err;
Expand All @@ -1885,11 +2067,15 @@ static void wc_end_poll_cq(struct ibv_cq_ex *current)
struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
struct hns_roce_context *ctx = to_hr_ctx(current->context);

if (unlikely(hns_roce_is_reset(ctx)))
goto end_poll_done;

if (cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)
*cq->db = cq->cons_index & RECORD_DB_CI_MASK;
else
update_cq_db(ctx, cq);

end_poll_done:
hns_roce_spin_unlock(&cq->hr_lock);
}

Expand Down
2 changes: 2 additions & 0 deletions providers/hns/hns_roce_u_hw_v2.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,5 +344,7 @@ void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp);
void hns_roce_attach_cq_ex_ops(struct ibv_cq_ex *cq_ex, uint64_t wc_flags);
int hns_roce_attach_qp_ex_ops(struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp);
void hns_roce_lock_cqs(struct ibv_qp *qp);
void hns_roce_unlock_cqs(struct ibv_qp *qp);

#endif /* _HNS_ROCE_U_HW_V2_H */
Loading

0 comments on commit 0092957

Please sign in to comment.