From: Valentine Fatiev <valentinef@nvidia.com>
Subject: [PATCH] BACKPORT: drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

Change-Id: Ie77d4916542912398f476d0bfffc87ee80b3ff84
---
 .../net/ethernet/mellanox/mlx5/core/en_rx.c   | 341 ++++++++++++++++--
 1 file changed, 316 insertions(+), 25 deletions(-)

--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -35,9 +35,14 @@
 #include <linux/tcp.h>
 #include <linux/bitmap.h>
 #include <net/ip6_checksum.h>
+#include <net/xdp.h>
+#ifdef HAVE_NET_PAGE_POOL_H
 #include <net/page_pool.h>
+#endif
 #include <net/inet_ecn.h>
+#ifdef HAVE_NET_GRO_H
 #include <net/gro.h>
+#endif
 #include <net/udp.h>
 #include <net/tcp.h>
 #include "en.h"
@@ -59,6 +64,15 @@
 #include "devlink.h"
 #include "en/devlink.h"
 #include "esw/ipsec.h"
+#include "en/txrx.h"
+
+static inline void mlx5e_set_skb_driver_xmit_more(struct sk_buff *skb,
+						  struct mlx5e_rq *rq,
+						  bool xmit_more)
+{
+	if (test_bit(MLX5E_RQ_STATE_SKB_XMIT_MORE, &rq->state) && xmit_more)
+		skb->cb[47] = MLX5_XMIT_MORE_SKB_CB;
+}
 
 static struct sk_buff *
 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
@@ -66,14 +80,18 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
 static struct sk_buff *
 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 				   u16 cqe_bcnt, u32 head_offset, u32 page_idx);
-static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
-static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
-static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
+static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool xmit_more);
+static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool xmit_more);
+#ifdef HAVE_SHAMPO_SUPPORT
+static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool xmit_more);
+#endif
 
 const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = {
 	.handle_rx_cqe       = mlx5e_handle_rx_cqe,
 	.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
+#ifdef HAVE_SHAMPO_SUPPORT
 	.handle_rx_cqe_mpwqe_shampo = mlx5e_handle_rx_cqe_mpwrq_shampo,
+#endif
 };
 
 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
@@ -193,9 +211,14 @@ static inline u32 mlx5e_decompress_cqes_
 			mlx5e_read_mini_arr_slot(wq, cqd, cqcc);
 
 		mlx5e_decompress_cqe_no_hash(rq, wq, cqcc);
+#ifdef HAVE_SHAMPO_SUPPORT
 		INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
 				mlx5e_handle_rx_cqe_mpwrq_shampo, mlx5e_handle_rx_cqe,
-				rq, &cqd->title);
+				rq, &cqd->title, i < cqe_count - 1);
+#else
+		INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
+				mlx5e_handle_rx_cqe, rq, &cqd->title, i < cqe_count - 1);
+#endif
 	}
 	mlx5e_cqes_update_owner(wq, cqcc - wq->cc);
 	wq->cc = cqcc;
@@ -215,9 +238,14 @@ static inline u32 mlx5e_decompress_cqes_
 	mlx5e_read_title_slot(rq, wq, cc);
 	mlx5e_read_mini_arr_slot(wq, cqd, cc + 1);
 	mlx5e_decompress_cqe(rq, wq, cc);
+#ifdef HAVE_SHAMPO_SUPPORT
 	INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
 			mlx5e_handle_rx_cqe_mpwrq_shampo, mlx5e_handle_rx_cqe,
-			rq, &cqd->title);
+			rq, &cqd->title, true);
+#else
+	INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
+			mlx5e_handle_rx_cqe, rq, &cqd->title, true);
+#endif
 	cqd->mini_arr_idx++;
 
 	return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1;
@@ -338,6 +366,11 @@ static inline bool mlx5e_rx_cache_extend
 	return true;
 }
 
+static inline bool mlx5e_page_is_reserved(struct page *page)
+{
+	return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
+}
+
 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
 				      struct mlx5e_dma_info *dma_info)
 {
@@ -350,8 +383,11 @@ static inline bool mlx5e_rx_cache_put(st
 			return false;
 		}
 	}
-
+#ifdef HAVE_DEV_PAGE_IS_REUSABLE
 	if (!dev_page_is_reusable(dma_info->page)) {
+#else
+	if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
+#endif
 		stats->cache_waive++;
 		return false;
 	}
@@ -416,18 +452,31 @@ static inline int mlx5e_page_alloc_pool(
 	if (mlx5e_rx_cache_get(rq, dma_info))
 		return 0;
 
-	dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
+#ifdef HAVE_NET_PAGE_POOL_H
+       dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
+#else
+	dma_info->page = dev_alloc_page();
+#endif
 	if (unlikely(!dma_info->page))
 		return -ENOMEM;
 
 	dma_info->refcnt_bias = 0;
 	page_ref_elev(dma_info);
 
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
 	dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE,
 					    rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
+#else
+	dma_info->addr = dma_map_page(rq->pdev, dma_info->page, 0,
+				      PAGE_SIZE, rq->buff.map_dir);
+#endif
 	if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
+#ifdef HAVE_NET_PAGE_POOL_H
 		page_pool_recycle_direct(rq->page_pool, dma_info->page);
 		page_ref_sub(dma_info->page, dma_info->refcnt_bias);
+#else
+		mlx5e_put_page(dma_info);
+#endif
 		dma_info->page = NULL;
 		return -ENOMEM;
 	}
@@ -438,22 +487,33 @@ static inline int mlx5e_page_alloc_pool(
 static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
 				   struct mlx5e_dma_info *dma_info)
 {
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
 	if (rq->xsk_pool)
+#else
+	if (rq->umem)
+#endif
 		return mlx5e_xsk_page_alloc_pool(rq, dma_info);
 	else
+#endif
 		return mlx5e_page_alloc_pool(rq, dma_info);
 }
 
 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
 {
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
 	dma_unmap_page_attrs(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir,
 			     DMA_ATTR_SKIP_CPU_SYNC);
+#else
+	dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
+#endif
 }
 
 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
 				struct mlx5e_dma_info *dma_info,
 				bool recycle)
 {
+#ifdef HAVE_NET_PAGE_POOL_H
 	if (likely(recycle)) {
 		if (mlx5e_rx_cache_put(rq, dma_info))
 			return;
@@ -463,22 +523,45 @@ void mlx5e_page_release_dynamic(struct m
 		page_pool_recycle_direct(rq->page_pool, dma_info->page);
 	} else {
 		mlx5e_page_dma_unmap(rq, dma_info);
+#ifdef HAVE_PAGE_POOL_RELEASE_PAGE
+		/* This call to page_pool_release_page should be part of
+		 * the base code, not backport, in the next rebase.
+		 */
 		page_pool_release_page(rq->page_pool, dma_info->page);
+#endif
 		mlx5e_put_page(dma_info);
 	}
+#else
+	if (likely(recycle) && mlx5e_rx_cache_put(rq, dma_info))
+		return;
+
+	mlx5e_page_dma_unmap(rq, dma_info);
+	mlx5e_put_page(dma_info);
+#endif
 }
 
 static inline void mlx5e_page_release(struct mlx5e_rq *rq,
 				      struct mlx5e_dma_info *dma_info,
 				      bool recycle)
 {
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
 	if (rq->xsk_pool)
+#else
+	if (rq->umem)
+#endif
+
 		/* The `recycle` parameter is ignored, and the page is always
 		 * put into the Reuse Ring, because there is no way to return
 		 * the page to the userspace when the interface goes down.
 		 */
+#ifdef HAVE_XSK_BUFF_ALLOC
 		xsk_buff_free(dma_info->xsk);
+#else
+		mlx5e_xsk_page_release(rq, dma_info);
+#endif
 	else
+#endif
 		mlx5e_page_release_dynamic(rq, dma_info, recycle);
 }
 
@@ -558,17 +641,28 @@ static int mlx5e_alloc_rx_wqes(struct ml
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 	int err;
 	int i;
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
+	struct xsk_buff_pool *xsk_pool = rq->xsk_pool;
+#else
+	struct xdp_umem *xsk_pool = rq->umem;
+#endif
 
-	if (rq->xsk_pool) {
+	if (xsk_pool) {
 		int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
 
 		/* Check in advance that we have enough frames, instead of
 		 * allocating one-by-one, failing and moving frames to the
 		 * Reuse Ring.
 		 */
-		if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, pages_desired)))
+#ifdef HAVE_XSK_BUFF_ALLOC
+		if (unlikely(!xsk_buff_can_alloc(xsk_pool, pages_desired)))
+#else
+		if (unlikely(!mlx5e_xsk_pages_enough_umem(rq, pages_desired)))
+#endif
 			return -ENOMEM;
 	}
+#endif
 
 	for (i = 0; i < wqe_bulk; i++) {
 		struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
@@ -631,6 +725,7 @@ static void mlx5e_mpwqe_page_release(str
 static void
 mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
 {
+#ifdef HAVE_XDP_SUPPORT
 	bool no_xdp_xmit;
 	struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
 	int i;
@@ -645,6 +740,13 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq,
 	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
 		if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
 			mlx5e_mpwqe_page_release(rq, &dma_info[i], recycle);
+#else
+	struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
+	int i;
+
+	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++)
+		mlx5e_mpwqe_page_release(rq, dma_info, recycle);
+#endif
 }
 
 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
@@ -658,7 +760,11 @@ static void mlx5e_post_rx_mpwqe(struct m
 	} while (--n);
 
 	/* ensure wqes are visible to device before updating doorbell record */
+#ifdef dma_wmb
 	dma_wmb();
+#else
+	wmb();
+#endif
 
 	mlx5_wq_ll_update_db_record(wq);
 
@@ -828,11 +934,22 @@ static int mlx5e_alloc_rx_mpwqe(struct m
 	/* Check in advance that we have enough frames, instead of allocating
 	 * one-by-one, failing and moving frames to the Reuse Ring.
 	 */
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
 	if (rq->xsk_pool &&
 	    unlikely(!xsk_buff_can_alloc(rq->xsk_pool, MLX5_MPWRQ_PAGES_PER_WQE))) {
+#elif defined(HAVE_XSK_BUFF_ALLOC)
+	if (rq->umem &&
+	    unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_PAGES_PER_WQE))) {
+
+#else
+	if (rq->umem &&
+	    unlikely(!mlx5e_xsk_pages_enough_umem(rq, MLX5_MPWRQ_PAGES_PER_WQE))) {
+#endif
 		err = -ENOMEM;
 		goto err;
 	}
+#endif
 
 	if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
 		err = mlx5e_alloc_rx_hd_mpwqe(rq);
@@ -851,7 +968,9 @@ static int mlx5e_alloc_rx_mpwqe(struct m
 		umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
 	}
 
+#ifdef HAVE_XDP_SUPPORT
 	bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
+#endif
 	wi->consumed_strides = 0;
 
 	umr_wqe->ctrl.opmod_idx_opcode =
@@ -943,8 +1062,10 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_
 	if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
 		return false;
 
+#ifdef HAVE_PAGE_POLL_NID_CHANGED
 	if (rq->page_pool)
 		page_pool_nid_changed(rq->page_pool, numa_mem_id());
+#endif
 
 	do {
 		u16 head = mlx5_wq_cyc_get_head(wq);
@@ -959,7 +1080,11 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_
 	} while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
 
 	/* ensure wqes are visible to device before updating doorbell record */
+#ifdef dma_wmb
 	dma_wmb();
+#else
+	wmb();
+#endif
 
 	mlx5_wq_cyc_update_db_record(wq);
 
@@ -991,7 +1116,7 @@ void mlx5e_free_icosq_descs(struct mlx5e
 		ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
 		wi = &sq->db.wqe_info[ci];
 		sqcc += wi->num_wqebbs;
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(HAVE_KTLS_RX_SUPPORT) && defined (CONFIG_MLX5_EN_TLS)
 		switch (wi->wqe_type) {
 		case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
 			mlx5e_ktls_handle_ctx_completion(wi);
@@ -1086,7 +1211,7 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *c
 			case MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR:
 				mlx5e_handle_shampo_hd_umr(wi->shampo, sq);
 				break;
-#ifdef CONFIG_MLX5_EN_TLS
+#if defined(HAVE_KTLS_RX_SUPPORT) && defined (CONFIG_MLX5_EN_TLS)
 			case MLX5E_ICOSQ_WQE_UMR_TLS:
 				break;
 			case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
@@ -1138,8 +1263,10 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_
 	if (likely(missing < UMR_WQE_BULK))
 		return false;
 
+#ifdef HAVE_PAGE_POLL_NID_CHANGED
 	if (rq->page_pool)
 		page_pool_nid_changed(rq->page_pool, numa_mem_id());
+#endif
 
 	head = rq->mpwqe.actual_wq_head;
 	i = missing;
@@ -1166,8 +1293,14 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_
 	 * the driver when it refills the Fill Ring.
 	 * 2. Otherwise, busy poll by rescheduling the NAPI poll.
 	 */
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
 	if (unlikely(alloc_err == -ENOMEM && rq->xsk_pool))
+#else
+	if (unlikely(alloc_err == -ENOMEM && rq->umem))
+#endif
 		return true;
+#endif
 
 	return false;
 }
@@ -1242,6 +1375,7 @@ static void mlx5e_lro_update_hdr(struct
 	}
 }
 
+#ifdef HAVE_SHAMPO_SUPPORT
 static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
 {
 	struct mlx5e_dma_info *last_head = &rq->mpwqe.shampo->info[header_index];
@@ -1377,6 +1511,7 @@ static void mlx5e_shampo_update_hdr(stru
 			mlx5e_shampo_update_ipv6_udp_hdr(rq, ipv6);
 	}
 }
+#endif /* HAVE_SHAMPO_SUPPORT */
 
 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
 				      struct sk_buff *skb)
@@ -1415,7 +1550,11 @@ static inline void mlx5e_enable_ecn(stru
 
 	ip = skb->data + network_depth;
 	rc = ((proto == htons(ETH_P_IP)) ? IP_ECN_set_ce((struct iphdr *)ip) :
+#ifdef HAVE_IP6_SET_CE_2_PARAMS
 					 IP6_ECN_set_ce(skb, (struct ipv6hdr *)ip));
+#else
+					 IP6_ECN_set_ce((struct ipv6hdr *)ip));
+#endif
 
 	rq->stats->ecn_mark += !!rc;
 }
@@ -1559,8 +1698,8 @@ csum_unnecessary:
 		   (cqe->hds_ip_ext & CQE_L4_OK))) {
 		skb->ip_summed = CHECKSUM_UNNECESSARY;
 		if (cqe_is_tunneled(cqe)) {
-			skb->csum_level = 1;
-			skb->encapsulation = 1;
+       		skb->csum_level = 1;
+       		skb->encapsulation = 1;
 			stats->csum_unnecessary_inner++;
 			return;
 		}
@@ -1582,6 +1721,10 @@ static inline void mlx5e_build_rx_skb(st
 	u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
 	struct mlx5e_rq_stats *stats = rq->stats;
 	struct net_device *netdev = rq->netdev;
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	u8 l4_hdr_type;
+#endif
 
 	skb->mac_len = ETH_HLEN;
 
@@ -1602,6 +1745,12 @@ static inline void mlx5e_build_rx_skb(st
 		stats->packets += lro_num_seg - 1;
 		stats->lro_packets++;
 		stats->lro_bytes += cqe_bcnt;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
+		/* Flush GRO to avoid OOO packets, since GSO bypasses the
+		 * GRO queue. This was fixed in dev_gro_receive() in kernel 4.10
+		 */
+		napi_gro_flush(rq->cq.napi, false);
+#endif
 	}
 
 	if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
@@ -1620,7 +1769,16 @@ static inline void mlx5e_build_rx_skb(st
 
 	skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
 
+#ifndef CONFIG_COMPAT_LRO_ENABLED_IPOIB
 	mlx5e_handle_csum(netdev, cqe, rq, skb, !!lro_num_seg);
+#else
+	l4_hdr_type = get_cqe_l4_hdr_type(cqe);
+	mlx5e_handle_csum(netdev, cqe, rq, skb,
+			  !!lro_num_seg ||
+			  (IS_SW_LRO(&priv->channels.params) &&
+			  (l4_hdr_type != CQE_L4_HDR_TYPE_NONE) &&
+			  (l4_hdr_type != CQE_L4_HDR_TYPE_UDP)));
+#endif
 	/* checking CE bit in cqe - MSB in ml_path field */
 	if (unlikely(cqe->ml_path & MLX5E_CE_BIT_MASK))
 		mlx5e_enable_ecn(rq, skb);
@@ -1631,6 +1789,7 @@ static inline void mlx5e_build_rx_skb(st
 		stats->mcast_packets++;
 }
 
+#ifdef HAVE_SHAMPO_SUPPORT
 static void mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq,
 					 struct mlx5_cqe64 *cqe,
 					 u32 cqe_bcnt,
@@ -1651,6 +1810,7 @@ static void mlx5e_shampo_complete_rx_cqe
 		rq->hw_gro_data->skb = NULL;
 	}
 }
+#endif
 
 static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
 					 struct mlx5_cqe64 *cqe,
@@ -1688,12 +1848,40 @@ struct sk_buff *mlx5e_build_linear_skb(s
 	return skb;
 }
 
+#ifdef HAVE_XDP_SUPPORT
 static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
 				u32 len, struct xdp_buff *xdp)
 {
+#ifdef HAVE_XDP_INIT_BUFF
 	xdp_init_buff(xdp, rq->buff.frame0_sz, &rq->xdp_rxq);
 	xdp_prepare_buff(xdp, va, headroom, len, false);
+#else
+	unsigned char *data = va + headroom;
+
+#ifdef HAVE_XDP_RXQ_INFO
+	xdp->rxq = &rq->xdp_rxq;
+#endif
+#ifdef HAVE_XDP_BUFF_HAS_FRAME_SZ
+	xdp->frame_sz = rq->buff.frame0_sz;
+#endif
+	xdp->data_hard_start = va;
+	xdp->data = data;
+	xdp->data_end = data + len;
+#ifdef HAVE_XDP_SET_DATA_META_INVALID
+        xdp_set_data_meta_invalid(xdp);
+#endif
+#endif
+}
+
+#if !defined(HAVE_XSK_BUFF_ALLOC) && defined(HAVE_XSK_ZERO_COPY_SUPPORT)
+void mlx5e_fill_xdp_buff_for_old_xsk(struct mlx5e_rq *rq, void *va, u16 headroom,
+				u32 len, struct xdp_buff *xdp, struct mlx5e_dma_info *di)
+{
+	mlx5e_fill_xdp_buff(rq, va, headroom, len, xdp);
+	xdp->handle = di->xsk.handle;
 }
+#endif
+#endif /* HAVE_XDP_SUPPORT */
 
 static struct sk_buff *
 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
@@ -1701,7 +1889,9 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_r
 {
 	struct mlx5e_dma_info *di = wi->di;
 	u16 rx_headroom = rq->buff.headroom;
+#ifdef HAVE_XDP_SUPPORT
 	struct xdp_buff xdp;
+#endif
 	struct sk_buff *skb;
 	void *va, *data;
 	u32 frag_size;
@@ -1715,11 +1905,13 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_r
 	net_prefetchw(va); /* xdp_frame data area */
 	net_prefetch(data);
 
+#ifdef HAVE_XDP_SUPPORT
 	mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
 	if (mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp))
 		return NULL; /* page/packet was consumed by XDP */
 
 	rx_headroom = xdp.data - xdp.data_hard_start;
+#endif
 	frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
 	skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
 	if (unlikely(!skb))
@@ -1794,8 +1986,12 @@ static void mlx5e_handle_rx_err_cqe(stru
 	rq->stats->wqe_err++;
 }
 
-static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			 bool xmit_more)
 {
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv = netdev_priv(rq->netdev);
+#endif
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 	struct mlx5e_wqe_frag_info *wi;
 	struct sk_buff *skb;
@@ -1828,12 +2024,19 @@ static void mlx5e_handle_rx_cqe(struct m
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
 
+	mlx5e_set_skb_driver_xmit_more(skb, rq, xmit_more);
+
 	if (mlx5e_cqe_regb_chain(cqe))
 		if (!mlx5e_tc_update_skb(cqe, skb)) {
 			dev_kfree_skb_any(skb);
 			goto free_wqe;
 		}
 
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	if (IS_SW_LRO(&priv->channels.params))
+		lro_receive_skb(&rq->sw_lro->lro_mgr, skb, NULL);
+	else
+#endif
 	napi_gro_receive(rq->cq.napi, skb);
 
 free_wqe:
@@ -1902,7 +2105,8 @@ static bool mlx5e_rep_lookup_and_update(
 	return true;
 }
 
-static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			     bool xmit_more)
 {
 	struct net_device *netdev = rq->netdev;
 	struct mlx5e_priv *priv = netdev_priv(netdev);
@@ -1954,7 +2158,8 @@ wq_cyc_pop:
 	mlx5_wq_cyc_pop(wq);
 }
 
-static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+					 bool xmit_more)
 {
 	u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
 	u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
@@ -2081,7 +2286,9 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
 	struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
 	u16 rx_headroom = rq->buff.headroom;
 	u32 cqe_bcnt32 = cqe_bcnt;
+#ifdef HAVE_XDP_SUPPORT
 	struct xdp_buff xdp;
+#endif
 	struct sk_buff *skb;
 	void *va, *data;
 	u32 frag_size;
@@ -2101,6 +2308,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
 	net_prefetchw(va); /* xdp_frame data area */
 	net_prefetch(data);
 
+#ifdef HAVE_XDP_SUPPORT
 	mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
 	if (mlx5e_xdp_handle(rq, di, &cqe_bcnt32, &xdp)) {
 		if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
@@ -2109,6 +2317,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
 	}
 
 	rx_headroom = xdp.data - xdp.data_hard_start;
+#endif
 	frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
 	skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
 	if (unlikely(!skb))
@@ -2120,6 +2329,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct m
 	return skb;
 }
 
+#ifdef HAVE_SHAMPO_SUPPORT
 static struct sk_buff *
 mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 			  struct mlx5_cqe64 *cqe, u16 header_index)
@@ -2217,7 +2427,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx
 	bitmap_clear(shampo->bitmap, header_index, 1);
 }
 
-static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool xmit_more)
 {
 	u16 data_bcnt		= mpwrq_get_cqe_byte_cnt(cqe) - cqe->shampo.header_size;
 	u16 header_index	= mlx5e_shampo_get_cqe_header_index(rq, cqe);
@@ -2301,9 +2511,14 @@ mpwrq_cqe_out:
 	mlx5e_free_rx_mpwqe(rq, wi, true);
 	mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
+#endif /* HAVE_SHAMPO_SUPPORT */
 
-static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			       bool xmit_more)
 {
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv = netdev_priv(rq->netdev);
+#endif
 	u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
 	u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
 	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
@@ -2342,12 +2557,19 @@ static void mlx5e_handle_rx_cqe_mpwrq(st
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
 
+	mlx5e_set_skb_driver_xmit_more(skb, rq, xmit_more);
+
 	if (mlx5e_cqe_regb_chain(cqe))
 		if (!mlx5e_tc_update_skb(cqe, skb)) {
 			dev_kfree_skb_any(skb);
 			goto mpwrq_cqe_out;
 		}
 
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	if (IS_SW_LRO(&priv->channels.params))
+		lro_receive_skb(&rq->sw_lro->lro_mgr, skb, NULL);
+	else
+#endif
 	napi_gro_receive(rq->cq.napi, skb);
 
 mpwrq_cqe_out:
@@ -2364,8 +2586,17 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq
 {
 	struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
 	struct mlx5_cqwq *cqwq = &cq->wq;
-	struct mlx5_cqe64 *cqe;
+	struct mlx5_cqe64 *cqe, *next_cqe;
 	int work_done = 0;
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv;
+#ifdef CONFIG_MLX5_CORE_IPOIB
+	if (MLX5_CAP_GEN(cq->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+		priv = mlx5i_epriv(rq->netdev);
+	else
+#endif
+		priv = netdev_priv(rq->netdev);
+#endif
 
 	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return 0;
@@ -2388,27 +2619,46 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq
 			work_done +=
 				mlx5e_decompress_cqes_start(rq, cqwq,
 							    budget - work_done);
+			if (work_done == budget)
+				break;
+			cqe = mlx5_cqwq_get_cqe(&cq->wq);
 			continue;
 		}
 
 		mlx5_cqwq_pop(cqwq);
 
+		next_cqe = mlx5_cqwq_get_cqe(&cq->wq);
+#ifdef HAVE_SHAMPO_SUPPORT
 		INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
 				mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo,
-				rq, cqe);
-	} while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
+				rq, cqe, next_cqe && (work_done & 0xf));
+#else
+		INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
+                                mlx5e_handle_rx_cqe, rq, cqe,
+                                next_cqe && (work_done & 0xf));
+#endif
+		cqe = next_cqe;
+	} while ((++work_done < budget) && cqe);
 
 out:
+#ifdef HAVE_SHAMPO_SUPPORT
 	if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) && rq->hw_gro_data->skb)
 		mlx5e_shampo_flush_skb(rq, NULL, false);
+#endif
 
+#ifdef HAVE_XDP_SUPPORT
 	if (rcu_access_pointer(rq->xdp_prog))
 		mlx5e_xdp_rx_poll_complete(rq);
+#endif
 
 	mlx5_cqwq_update_db_record(cqwq);
 
 	/* ensure cq space is freed before enabling more cqes */
 	wmb();
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	if (IS_SW_LRO(&priv->channels.params))
+		lro_flush_all(&rq->sw_lro->lro_mgr);
+#endif
 
 	return work_done;
 }
@@ -2434,6 +2684,9 @@ static inline void mlx5i_complete_rx_cqe
 	u32 qpn;
 	u8 *dgid;
 	u8 g;
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+       struct mlx5e_priv *parent_priv = mlx5i_epriv(rq->netdev);
+#endif
 
 	qpn = be32_to_cpu(cqe->sop_drop_qpn) & 0xffffff;
 	netdev = mlx5i_pkey_get_netdev(rq->netdev, qpn);
@@ -2476,6 +2729,12 @@ static inline void mlx5i_complete_rx_cqe
 
 	skb->protocol = *((__be16 *)(skb->data));
 
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	if (parent_priv->netdev->features & NETIF_F_LRO) {
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+	} else
+#endif
+
 	if ((netdev->features & NETIF_F_RXCSUM) &&
 	    (likely((cqe->hds_ip_ext & CQE_L3_OK) &&
 		    (cqe->hds_ip_ext & CQE_L4_OK)))) {
@@ -2511,8 +2770,12 @@ static inline void mlx5i_complete_rx_cqe
 	}
 }
 
-static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			 bool xmit_more)
 {
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv = mlx5i_epriv(rq->netdev);
+#endif
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 	struct mlx5e_wqe_frag_info *wi;
 	struct sk_buff *skb;
@@ -2540,6 +2803,12 @@ static void mlx5i_handle_rx_cqe(struct m
 		dev_kfree_skb_any(skb);
 		goto wq_free_wqe;
 	}
+	mlx5e_set_skb_driver_xmit_more(skb, rq, xmit_more);
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	if (priv->netdev->features & NETIF_F_LRO)
+		lro_receive_skb(&rq->sw_lro->lro_mgr, skb, NULL);
+	else
+#endif
 	napi_gro_receive(rq->cq.napi, skb);
 
 wq_free_wqe:
@@ -2555,8 +2824,12 @@ const struct mlx5e_rx_handlers mlx5i_rx_
 
 #ifdef CONFIG_MLX5_EN_IPSEC
 
-static void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			       bool xmit_more)
 {
+#ifdef CONFIG_COMPAT_LRO_ENABLED_IPOIB
+	struct mlx5e_priv *priv = mlx5i_epriv(rq->netdev);
+#endif
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 	struct mlx5e_wqe_frag_info *wi;
 	struct sk_buff *skb;
@@ -2584,6 +2857,7 @@ static void mlx5e_ipsec_handle_rx_cqe(st
 		goto wq_free_wqe;
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
+	mlx5e_set_skb_driver_xmit_more(skb, rq, xmit_more);
 	napi_gro_receive(rq->cq.napi, skb);
 
 wq_free_wqe:
@@ -2606,11 +2880,19 @@ int mlx5e_rq_set_handlers(struct mlx5e_r
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
 		rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
 			mlx5e_xsk_skb_from_cqe_mpwrq_linear :
 			mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
 				mlx5e_skb_from_cqe_mpwrq_linear :
 				mlx5e_skb_from_cqe_mpwrq_nonlinear;
+#else
+		rq->mpwqe.skb_from_cqe_mpwrq =
+			mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
+			mlx5e_skb_from_cqe_mpwrq_linear :
+			mlx5e_skb_from_cqe_mpwrq_nonlinear;
+#endif
+
 		rq->post_wqes = mlx5e_post_rx_mpwqes;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 
@@ -2634,11 +2916,17 @@ int mlx5e_rq_set_handlers(struct mlx5e_r
 
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		rq->wqe.skb_from_cqe = xsk ?
+#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
+                rq->wqe.skb_from_cqe = xsk ?
 			mlx5e_xsk_skb_from_cqe_linear :
 			mlx5e_rx_is_linear_skb(params, NULL) ?
 				mlx5e_skb_from_cqe_linear :
 				mlx5e_skb_from_cqe_nonlinear;
+#else
+		rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ?
+			mlx5e_skb_from_cqe_linear :
+			mlx5e_skb_from_cqe_nonlinear;
+#endif
 		rq->post_wqes = mlx5e_post_rx_wqes;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
@@ -2658,7 +2946,9 @@ int mlx5e_rq_set_handlers(struct mlx5e_r
 	return 0;
 }
 
-static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+#ifdef HAVE_DEVLINK_TRAP_SUPPORT
+static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+		bool xmit_more)
 {
 	struct mlx5e_priv *priv = netdev_priv(rq->netdev);
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
@@ -2704,3 +2994,4 @@ void mlx5e_rq_set_trap_handlers(struct m
 	rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 	rq->handle_rx_cqe = mlx5e_trap_handle_rx_cqe;
 }
+#endif /* HAVE_DEVLINK_TRAP_SUPPORT */
