net/mlx5e: CQE compression

[cascardo/linux.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_rx.c

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 23adfe2..c8b8d45 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -42,6 +42,143 @@ static inline bool mlx5e_rx_hw_stamp(struct mlx5e_tstamp *tstamp)
        return tstamp->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL;
 }
 
+static inline void mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cqcc,
+                                      void *data)
+{
+       u32 ci = cqcc & cq->wq.sz_m1;
+
+       memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, ci), sizeof(struct mlx5_cqe64));
+}
+
+static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
+                                        struct mlx5e_cq *cq, u32 cqcc)
+{
+       mlx5e_read_cqe_slot(cq, cqcc, &cq->title);
+       cq->decmprs_left        = be32_to_cpu(cq->title.byte_cnt);
+       cq->decmprs_wqe_counter = be16_to_cpu(cq->title.wqe_counter);
+       rq->stats.cqe_compress_blks++;
+}
+
+static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc)
+{
+       mlx5e_read_cqe_slot(cq, cqcc, cq->mini_arr);
+       cq->mini_arr_idx = 0;
+}
+
+static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)
+{
+       u8 op_own = (cqcc >> cq->wq.log_sz) & 1;
+       u32 wq_sz = 1 << cq->wq.log_sz;
+       u32 ci = cqcc & cq->wq.sz_m1;
+       u32 ci_top = min_t(u32, wq_sz, ci + n);
+
+       for (; ci < ci_top; ci++, n--) {
+               struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, ci);
+
+               cqe->op_own = op_own;
+       }
+
+       if (unlikely(ci == wq_sz)) {
+               op_own = !op_own;
+               for (ci = 0; ci < n; ci++) {
+                       struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, ci);
+
+                       cqe->op_own = op_own;
+               }
+       }
+}
+
+static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
+                                       struct mlx5e_cq *cq, u32 cqcc)
+{
+       u16 wqe_cnt_step;
+
+       cq->title.byte_cnt     = cq->mini_arr[cq->mini_arr_idx].byte_cnt;
+       cq->title.check_sum    = cq->mini_arr[cq->mini_arr_idx].checksum;
+       cq->title.op_own      &= 0xf0;
+       cq->title.op_own      |= 0x01 & (cqcc >> cq->wq.log_sz);
+       cq->title.wqe_counter  = cpu_to_be16(cq->decmprs_wqe_counter);
+
+       wqe_cnt_step =
+               rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
+               mpwrq_get_cqe_consumed_strides(&cq->title) : 1;
+       cq->decmprs_wqe_counter =
+               (cq->decmprs_wqe_counter + wqe_cnt_step) & rq->wq.sz_m1;
+}
+
+static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
+                                               struct mlx5e_cq *cq, u32 cqcc)
+{
+       mlx5e_decompress_cqe(rq, cq, cqcc);
+       cq->title.rss_hash_type   = 0;
+       cq->title.rss_hash_result = 0;
+}
+
+static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
+                                            struct mlx5e_cq *cq,
+                                            int update_owner_only,
+                                            int budget_rem)
+{
+       u32 cqcc = cq->wq.cc + update_owner_only;
+       u32 cqe_count;
+       u32 i;
+
+       cqe_count = min_t(u32, cq->decmprs_left, budget_rem);
+
+       for (i = update_owner_only; i < cqe_count;
+            i++, cq->mini_arr_idx++, cqcc++) {
+               if (unlikely(cq->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE))
+                       mlx5e_read_mini_arr_slot(cq, cqcc);
+
+               mlx5e_decompress_cqe_no_hash(rq, cq, cqcc);
+               rq->handle_rx_cqe(rq, &cq->title);
+       }
+       mlx5e_cqes_update_owner(cq, cq->wq.cc, cqcc - cq->wq.cc);
+       cq->wq.cc = cqcc;
+       cq->decmprs_left -= cqe_count;
+       rq->stats.cqe_compress_pkts += cqe_count;
+
+       return cqe_count;
+}
+
+static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
+                                             struct mlx5e_cq *cq,
+                                             int budget_rem)
+{
+       mlx5e_read_title_slot(rq, cq, cq->wq.cc);
+       mlx5e_read_mini_arr_slot(cq, cq->wq.cc + 1);
+       mlx5e_decompress_cqe(rq, cq, cq->wq.cc);
+       rq->handle_rx_cqe(rq, &cq->title);
+       cq->mini_arr_idx++;
+
+       return mlx5e_decompress_cqes_cont(rq, cq, 1, budget_rem) - 1;
+}
+
+void mlx5e_modify_rx_cqe_compression(struct mlx5e_priv *priv, bool val)
+{
+       bool was_opened;
+
+       if (!MLX5_CAP_GEN(priv->mdev, cqe_compression))
+               return;
+
+       mutex_lock(&priv->state_lock);
+
+       if (priv->params.rx_cqe_compress == val)
+               goto unlock;
+
+       was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+       if (was_opened)
+               mlx5e_close_locked(priv->netdev);
+
+       priv->params.rx_cqe_compress = val;
+
+       if (was_opened)
+               mlx5e_open_locked(priv->netdev);
+
+unlock:
+       mutex_unlock(&priv->state_lock);
+}
+
 int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
 {
        struct sk_buff *skb;
@@ -738,14 +875,24 @@ mpwrq_cqe_out:
 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 {
        struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
-       int work_done;
+       int work_done = 0;
 
-       for (work_done = 0; work_done < budget; work_done++) {
+       if (cq->decmprs_left)
+               work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget);
+
+       for (; work_done < budget; work_done++) {
                struct mlx5_cqe64 *cqe = mlx5e_get_cqe(cq);
 
                if (!cqe)
                        break;
 
+               if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
+                       work_done +=
+                               mlx5e_decompress_cqes_start(rq, cq,
+                                                           budget - work_done);
+                       continue;
+               }
+
                mlx5_cqwq_pop(&cq->wq);
 
                rq->handle_rx_cqe(rq, cqe);